Skip to content

Commit

Permalink
Merge pull request #4450 from olljanat/support-ipv6-only
Browse files Browse the repository at this point in the history 
Add partial support for IPv6 only mode
  • Loading branch information
@rbrtbnfgl
rbrtbnfgl committed Mar 8, 2022
2 parents 44c5374 + 966f4d6 commit 3fabc07
Show file tree
Hide file tree
Showing 6 changed files with 187 additions and 23 deletions.
11 changes: 6 additions & 5 deletions pkg/agent/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
Expand Down Expand Up @@ -466,18 +465,20 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.Certificate = servingCert

nodeConfig.AgentConfig.NodeIPs = nodeIPs
nodeIP, err := util.GetFirst4(nodeIPs)
nodeIP, listenAddress, _, err := util.GetFirstIP(nodeIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-ip")
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
}
nodeConfig.AgentConfig.NodeIP = nodeIP.String()
nodeConfig.AgentConfig.ListenAddress = listenAddress
nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs

// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
// unless only IPv6 address given
if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 {
nodeExternalIP, err := util.GetFirst4(nodeConfig.AgentConfig.NodeExternalIPs)
nodeExternalIP, _, _, err := util.GetFirstIP(nodeConfig.AgentConfig.NodeExternalIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-external-ip")
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip")
}
nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String()
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/agent/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,10 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
if err != nil {
return errors.Wrap(err, "failed to validate node-ip")
}
serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR)
clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR)

enableIPv6 := dualCluster || dualService || dualNode
enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6 || clusterIPv6
conntrackConfig, err := getConntrackConfig(nodeConfig)
if err != nil {
return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
Expand Down
52 changes: 39 additions & 13 deletions pkg/cli/server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,17 +200,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont

if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
// ignoring the error here is fine since etcd will fall back to the interface's IPv4 address
serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
serverConfig.ControlConfig.PrivateIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
}

// if not set, try setting advertise-ip from agent node-external-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeExternalIP)
serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeExternalIP)
}

// if not set, try setting advertise-ip from agent node-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
}

// if we ended up with any advertise-ips, ensure they're added to the SAN list;
Expand All @@ -228,14 +228,19 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
return err
}
serverConfig.ControlConfig.ServerNodeName = nodeName
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "localhost", nodeName)
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName)
for _, ip := range nodeIPs {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String())
}

// configure ClusterIPRanges
_, _, IPv6only, _ := util.GetFirstIP(nodeIPs)
if len(cmds.ServerConfig.ClusterCIDR) == 0 {
cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16")
clusterCIDR := "10.42.0.0/16"
if IPv6only {
clusterCIDR = "fd:42::/56"
}
cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR)
}
for _, cidr := range cmds.ServerConfig.ClusterCIDR {
for _, v := range strings.Split(cidr, ",") {
Expand All @@ -248,15 +253,20 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}

// set ClusterIPRange to the first IPv4 block, for legacy clients
clusterIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ClusterIPRanges)
// unless only IPv6 range given
clusterIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ClusterIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-cidr")
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr")
}
serverConfig.ControlConfig.ClusterIPRange = clusterIPRange

// configure ServiceIPRanges
if len(cmds.ServerConfig.ServiceCIDR) == 0 {
cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16")
serviceCIDR := "10.43.0.0/16"
if IPv6only {
serviceCIDR = "fd:43::/112"
}
cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR)
}
for _, cidr := range cmds.ServerConfig.ServiceCIDR {
for _, v := range strings.Split(cidr, ",") {
Expand All @@ -269,9 +279,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}

// set ServiceIPRange to the first IPv4 block, for legacy clients
serviceIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ServiceIPRanges)
// unless only IPv6 range given
serviceIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ServiceIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 service-cidr")
return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr")
}
serverConfig.ControlConfig.ServiceIPRange = serviceIPRange

Expand All @@ -289,7 +300,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont

// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
// If there are no IPv4 ServiceCIDRs, an error will be raised.
// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
// If neither of IPv4 or IPv6 are found an error is raised.
if len(cmds.ServerConfig.ClusterDNS) == 0 {
clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10)
if err != nil {
Expand All @@ -308,9 +320,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
}
// Set ClusterDNS to the first IPv4 address, for legacy clients
clusterDNS, err := util.GetFirst4(serverConfig.ControlConfig.ClusterDNSs)
// unless only IPv6 range given
clusterDNS, _, _, err := util.GetFirstIP(serverConfig.ControlConfig.ClusterDNSs)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-dns address")
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address")
}
serverConfig.ControlConfig.ClusterDNS = clusterDNS
}
Expand Down Expand Up @@ -446,6 +459,9 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
ip := serverConfig.ControlConfig.BindAddress
if ip == "" {
ip = "127.0.0.1"
if IPv6only {
ip = "[::1]"
}
}

url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort)
Expand Down Expand Up @@ -513,6 +529,16 @@ func validateNetworkConfiguration(serverConfig server.Config) error {
return errors.New("dual-stack cluster-dns is not supported")
}

IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(serverConfig.ControlConfig.ServiceIPRanges)
if IPv6OnlyService {
if serverConfig.ControlConfig.DisableNPC == false {
return errors.New("network policy enforcement is not compatible with IPv6 only operation; server must be restarted with --disable-network-policy")
}
if serverConfig.ControlConfig.FlannelBackend != config.FlannelBackendNone {
return errors.New("Flannel is not compatible with IPv6 only operation; server must be restarted with --flannel-backend=none")
}
}

return nil
}

Expand Down
14 changes: 12 additions & 2 deletions pkg/daemons/agent/agent_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}

func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
"conntrack-max-per-core": "0",
Expand All @@ -56,8 +61,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}

func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
Expand Down
14 changes: 12 additions & 2 deletions pkg/daemons/agent/agent_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}

func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "kernelspace",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
}
Expand All @@ -46,8 +51,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}

func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
Expand Down
115 changes: 115 additions & 0 deletions pkg/util/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,46 @@ func JoinIP4Nets(elems []*net.IPNet) string {
return strings.Join(strs, ",")
}

// GetFirst6 returns the first IPv6 address from the list of IP addresses.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6(elems []net.IP) (net.IP, error) {
for _, elem := range elems {
if elem == nil || elem.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 address found")
}

// GetFirst6Net returns the first IPv4 network from the list of IP networks.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
for _, elem := range elems {
if elem == nil || elem.IP.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 CIDRs found")
}

// GetFirst6String returns the first IPv6 address from a list of IP address strings.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6String(elems []string) (string, error) {
ips := []net.IP{}
for _, elem := range elems {
for _, v := range strings.Split(elem, ",") {
ips = append(ips, net.ParseIP(v))
}
}
ip, err := GetFirst6(ips)
if err != nil {
return "", err
}
return ip.String(), nil
}

// JoinIP6Nets stringifies and joins a list of IPv6 networks with commas.
func JoinIP6Nets(elems []*net.IPNet) string {
var strs []string
Expand Down Expand Up @@ -141,3 +181,78 @@ func ParseStringSliceToIPs(s cli.StringSlice) ([]net.IP, error) {

return ips, nil
}

// GetFirstIP returns the first IPv4 address from the list of IP addresses.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
// Additionally matching listen address and IP version is returned.
func GetFirstIP(nodeIPs []net.IP) (net.IP, string, bool, error) {
nodeIP, err := GetFirst4(nodeIPs)
ListenAddress := "0.0.0.0"
IPv6only := false
if err != nil {
nodeIP, err = GetFirst6(nodeIPs)
if err != nil {
return nil, "", false, err
}
ListenAddress = "::"
IPv6only = true
}
return nodeIP, ListenAddress, IPv6only, nil
}

// GetFirstNet returns the first IPv4 network from the list of IP networks.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstNet(elems []*net.IPNet) (*net.IPNet, error) {
serviceIPRange, err := GetFirst4Net(elems)
if err != nil {
serviceIPRange, err = GetFirst6Net(elems)
if err != nil {
return nil, err
}
}
return serviceIPRange, nil
}

// GetFirstString returns the first IP4 address from a list of IP address strings.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstString(elems []string) (string, bool, error) {
ip, err := GetFirst4String(elems)
IPv6only := false
if err != nil {
ip, err = GetFirst6String(elems)
if err != nil {
return "", false, err
}
IPv6only = true
}
return ip, IPv6only, nil
}

// IsIPv6OnlyCIDRs returns if
// - all are valid cidrs
// - at least one cidr from v6 family is found
// - v4 family cidr is not found
func IsIPv6OnlyCIDRs(cidrs []*net.IPNet) (bool, error) {
v4Found := false
v6Found := false
for _, cidr := range cidrs {
if cidr == nil {
return false, fmt.Errorf("cidr %v is invalid", cidr)
}

if v4Found && v6Found {
continue
}

if cidr.IP != nil && cidr.IP.To4() == nil {
v6Found = true
continue
}
v4Found = true
}

return !v4Found && v6Found, nil
}

0 comments on commit 3fabc07

Please sign in to comment.