Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add partial support for IPv6 only mode #4450

Merged
merged 1 commit into from
Mar 8, 2022
Merged

Add partial support for IPv6 only mode #4450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions pkg/agent/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
Expand Down Expand Up @@ -466,18 +465,20 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.Certificate = servingCert

nodeConfig.AgentConfig.NodeIPs = nodeIPs
nodeIP, err := util.GetFirst4(nodeIPs)
nodeIP, listenAddress, _, err := util.GetFirstIP(nodeIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-ip")
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
}
nodeConfig.AgentConfig.NodeIP = nodeIP.String()
nodeConfig.AgentConfig.ListenAddress = listenAddress
nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs

// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
// unless only IPv6 address given
if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 {
nodeExternalIP, err := util.GetFirst4(nodeConfig.AgentConfig.NodeExternalIPs)
nodeExternalIP, _, _, err := util.GetFirstIP(nodeConfig.AgentConfig.NodeExternalIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-external-ip")
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip")
}
nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String()
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/agent/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,10 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
if err != nil {
return errors.Wrap(err, "failed to validate node-ip")
}
serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR)
clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR)

enableIPv6 := dualCluster || dualService || dualNode
enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6 || clusterIPv6
conntrackConfig, err := getConntrackConfig(nodeConfig)
if err != nil {
return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
Expand Down
52 changes: 39 additions & 13 deletions pkg/cli/server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,17 +198,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont

if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
// ignoring the error here is fine since etcd will fall back to the interface's IPv4 address
serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
serverConfig.ControlConfig.PrivateIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
}

// if not set, try setting advertise-ip from agent node-external-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeExternalIP)
serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeExternalIP)
}

// if not set, try setting advertise-ip from agent node-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
}

// if we ended up with any advertise-ips, ensure they're added to the SAN list;
Expand All @@ -226,14 +226,19 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
return err
}
serverConfig.ControlConfig.ServerNodeName = nodeName
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "localhost", nodeName)
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName)
for _, ip := range nodeIPs {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String())
}

// configure ClusterIPRanges
_, _, IPv6only, _ := util.GetFirstIP(nodeIPs)
if len(cmds.ServerConfig.ClusterCIDR) == 0 {
cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16")
clusterCIDR := "10.42.0.0/16"
if IPv6only {
clusterCIDR = "fd:42::/56"
}
cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR)
}
for _, cidr := range cmds.ServerConfig.ClusterCIDR {
for _, v := range strings.Split(cidr, ",") {
Expand All @@ -246,15 +251,20 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}

// set ClusterIPRange to the first IPv4 block, for legacy clients
clusterIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ClusterIPRanges)
// unless only IPv6 range given
clusterIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ClusterIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-cidr")
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr")
}
serverConfig.ControlConfig.ClusterIPRange = clusterIPRange

// configure ServiceIPRanges
if len(cmds.ServerConfig.ServiceCIDR) == 0 {
cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16")
serviceCIDR := "10.43.0.0/16"
if IPv6only {
serviceCIDR = "fd:43::/112"
}
cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR)
}
for _, cidr := range cmds.ServerConfig.ServiceCIDR {
for _, v := range strings.Split(cidr, ",") {
Expand All @@ -267,9 +277,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}

// set ServiceIPRange to the first IPv4 block, for legacy clients
serviceIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ServiceIPRanges)
// unless only IPv6 range given
serviceIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ServiceIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 service-cidr")
return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr")
}
serverConfig.ControlConfig.ServiceIPRange = serviceIPRange

Expand All @@ -287,7 +298,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont

// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
// If there are no IPv4 ServiceCIDRs, an error will be raised.
// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
// If neither of IPv4 or IPv6 are found an error is raised.
if len(cmds.ServerConfig.ClusterDNS) == 0 {
clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10)
if err != nil {
Expand All @@ -306,9 +318,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
}
// Set ClusterDNS to the first IPv4 address, for legacy clients
clusterDNS, err := util.GetFirst4(serverConfig.ControlConfig.ClusterDNSs)
// unless only IPv6 range given
clusterDNS, _, _, err := util.GetFirstIP(serverConfig.ControlConfig.ClusterDNSs)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-dns address")
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address")
}
serverConfig.ControlConfig.ClusterDNS = clusterDNS
}
Expand Down Expand Up @@ -457,6 +470,9 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
ip := serverConfig.ControlConfig.BindAddress
if ip == "" {
ip = "127.0.0.1"
if IPv6only {
ip = "[::1]"
}
}

url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort)
Expand Down Expand Up @@ -517,6 +533,16 @@ func validateNetworkConfiguration(serverConfig server.Config) error {
return errors.New("dual-stack cluster-dns is not supported")
}

IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(serverConfig.ControlConfig.ServiceIPRanges)
if IPv6OnlyService {
if serverConfig.ControlConfig.DisableNPC == false {
return errors.New("network policy enforcement is not compatible with IPv6 only operation; server must be restarted with --disable-network-policy")
}
if serverConfig.ControlConfig.FlannelBackend != config.FlannelBackendNone {
return errors.New("Flannel is not compatible with IPv6 only operation; server must be restarted with --flannel-backend=none")
}
}

return nil
}

Expand Down
14 changes: 12 additions & 2 deletions pkg/daemons/agent/agent_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}

func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
"conntrack-max-per-core": "0",
Expand All @@ -55,8 +60,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}

func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
Expand Down
14 changes: 12 additions & 2 deletions pkg/daemons/agent/agent_windows.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}

func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "kernelspace",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
}
Expand All @@ -45,8 +50,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}

func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
Expand Down
115 changes: 115 additions & 0 deletions pkg/util/net.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,46 @@ func JoinIP4Nets(elems []*net.IPNet) string {
return strings.Join(strs, ",")
}

// GetFirst6 returns the first IPv6 address from the list of IP addresses.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6(elems []net.IP) (net.IP, error) {
for _, elem := range elems {
if elem == nil || elem.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 address found")
}

// GetFirst6Net returns the first IPv4 network from the list of IP networks.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
for _, elem := range elems {
if elem == nil || elem.IP.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 CIDRs found")
}

// GetFirst6String returns the first IPv6 address from a list of IP address strings.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6String(elems []string) (string, error) {
ips := []net.IP{}
for _, elem := range elems {
for _, v := range strings.Split(elem, ",") {
ips = append(ips, net.ParseIP(v))
}
}
ip, err := GetFirst6(ips)
if err != nil {
return "", err
}
return ip.String(), nil
}

// JoinIP6Nets stringifies and joins a list of IPv6 networks with commas.
func JoinIP6Nets(elems []*net.IPNet) string {
var strs []string
Expand Down Expand Up @@ -141,3 +181,78 @@ func ParseStringSliceToIPs(s cli.StringSlice) ([]net.IP, error) {

return ips, nil
}

// GetFirstIP returns the first IPv4 address from the list of IP addresses.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
// Additionally matching listen address and IP version is returned.
func GetFirstIP(nodeIPs []net.IP) (net.IP, string, bool, error) {
nodeIP, err := GetFirst4(nodeIPs)
ListenAddress := "0.0.0.0"
IPv6only := false
if err != nil {
nodeIP, err = GetFirst6(nodeIPs)
if err != nil {
return nil, "", false, err
}
ListenAddress = "::"
IPv6only = true
}
return nodeIP, ListenAddress, IPv6only, nil
}

// GetFirstNet returns the first IPv4 network from the list of IP networks.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstNet(elems []*net.IPNet) (*net.IPNet, error) {
serviceIPRange, err := GetFirst4Net(elems)
if err != nil {
serviceIPRange, err = GetFirst6Net(elems)
if err != nil {
return nil, err
}
}
return serviceIPRange, nil
}

// GetFirstString returns the first IP4 address from a list of IP address strings.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstString(elems []string) (string, bool, error) {
ip, err := GetFirst4String(elems)
IPv6only := false
if err != nil {
ip, err = GetFirst6String(elems)
if err != nil {
return "", false, err
}
IPv6only = true
}
return ip, IPv6only, nil
}

// IsIPv6OnlyCIDRs returns if
// - all are valid cidrs
// - at least one cidr from v6 family is found
// - v4 family cidr is not found
func IsIPv6OnlyCIDRs(cidrs []*net.IPNet) (bool, error) {
v4Found := false
v6Found := false
for _, cidr := range cidrs {
if cidr == nil {
return false, fmt.Errorf("cidr %v is invalid", cidr)
}

if v4Found && v6Found {
continue
}

if cidr.IP != nil && cidr.IP.To4() == nil {
v6Found = true
continue
}
v4Found = true
}

return !v4Found && v6Found, nil
}