Skip to content

Commit

Permalink
Merge pull request #79386 from khenidak/phase2-dualstack
Browse files Browse the repository at this point in the history
Phase 2 dualstack
  • Loading branch information
k8s-ci-robot committed Aug 29, 2019
2 parents ca5babc + c27e0b0 commit 550fb1b
Show file tree
Hide file tree
Showing 61 changed files with 3,223 additions and 1,150 deletions.
1 change: 1 addition & 0 deletions api/api-rules/violation_exceptions.list
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,K
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,ConcurrentNamespaceSyncs
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,NamespaceSyncPeriod
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,NodeCIDRMaskSize
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,SecondaryServiceCIDR
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,ServiceCIDR
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeLifecycleControllerConfiguration,EnableTaintManager
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeLifecycleControllerConfiguration,LargeClusterSizeThreshold
Expand Down
4 changes: 4 additions & 0 deletions api/openapi-spec/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions cmd/kube-apiserver/app/options/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ go_library(
"//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
)

Expand All @@ -43,22 +44,26 @@ go_test(
srcs = [
"globalflags_test.go",
"options_test.go",
"validation_test.go",
],
embed = [":go_default_library"],
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubeapiserver/options:go_default_library",
"//pkg/kubelet/client:go_default_library",
"//pkg/master/reconcilers:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library",
"//staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic:go_default_library",
"//staging/src/k8s.io/apiserver/plugin/pkg/audit/truncate:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
"//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
],
)
Expand Down
19 changes: 13 additions & 6 deletions cmd/kube-apiserver/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,16 @@ type ServerRunOptions struct {
KubeletConfig kubeletclient.KubeletClientConfig
KubernetesServiceNodePort int
MaxConnectionBytesPerSec int64
ServiceClusterIPRange net.IPNet // TODO: make this a list
ServiceNodePortRange utilnet.PortRange
SSHKeyfile string
SSHUser string
// ServiceClusterIPRange is mapped to input provided by user
ServiceClusterIPRanges string
//PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results
// of parsing ServiceClusterIPRange into actual values
PrimaryServiceClusterIPRange net.IPNet
SecondaryServiceClusterIPRange net.IPNet

ServiceNodePortRange utilnet.PortRange
SSHKeyfile string
SSHUser string

ProxyClientCertFile string
ProxyClientKeyFile string
Expand Down Expand Up @@ -114,7 +120,7 @@ func NewServerRunOptions() *ServerRunOptions {
},
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
}
s.ServiceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
s.ServiceClusterIPRanges = kubeoptions.DefaultServiceIPCIDR.String()

// Overwrite the default for storage data format.
s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
Expand Down Expand Up @@ -179,7 +185,8 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
"of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+
"service will be of type ClusterIP.")

fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+
// TODO (khenidak) change documentation as we move IPv6DualStack feature from ALPHA to BETA
fs.StringVar(&s.ServiceClusterIPRanges, "service-cluster-ip-range", s.ServiceClusterIPRanges, ""+
"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
"overlap with any IP ranges assigned to nodes for pods.")

Expand Down
2 changes: 1 addition & 1 deletion cmd/kube-apiserver/app/options/options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func TestAddFlags(t *testing.T) {
// This is a snapshot of expected options parsed by args.
expected := &ServerRunOptions{
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
ServiceClusterIPRange: kubeoptions.DefaultServiceIPCIDR,
ServiceClusterIPRanges: kubeoptions.DefaultServiceIPCIDR.String(),
MasterCount: 5,
EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
AllowPrivileged: false,
Expand Down
49 changes: 46 additions & 3 deletions cmd/kube-apiserver/app/options/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,69 @@ package options
import (
"errors"
"fmt"
"net"
"strings"

apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
utilfeature "k8s.io/apiserver/pkg/util/feature"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/features"
netutils "k8s.io/utils/net"
)

// TODO: Longer term we should read this from some config store, rather than a flag.
// validateClusterIPFlags is expected to be called after Complete()
func validateClusterIPFlags(options *ServerRunOptions) []error {
var errs []error

if options.ServiceClusterIPRange.IP == nil {
errs = append(errs, errors.New("no --service-cluster-ip-range specified"))
// validate that primary has been processed by user provided values or it has been defaulted
if options.PrimaryServiceClusterIPRange.IP == nil {
errs = append(errs, errors.New("--service-cluster-ip-range must contain at least one valid cidr"))
}
var ones, bits = options.ServiceClusterIPRange.Mask.Size()

serviceClusterIPRangeList := strings.Split(options.ServiceClusterIPRanges, ",")
if len(serviceClusterIPRangeList) > 2 {
errs = append(errs, errors.New("--service-cluster-ip-range must not contain more than two entries"))
}

// Complete() expected to have set Primary* and Secondary*
// primary CIDR validation
var ones, bits = options.PrimaryServiceClusterIPRange.Mask.Size()
if bits-ones > 20 {
errs = append(errs, errors.New("specified --service-cluster-ip-range is too large"))
}

// Secondary IP validation
secondaryServiceClusterIPRangeUsed := (options.SecondaryServiceClusterIPRange.IP != nil)
if secondaryServiceClusterIPRangeUsed && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
errs = append(errs, fmt.Errorf("--secondary-service-cluster-ip-range can only be used if %v feature is enabled", string(features.IPv6DualStack)))
}

// note: While the cluster might be dualstack (i.e. pods with multiple IPs), the user may choose
// to only ingress traffic within and into the cluster on one IP family only. this family is decided
// by the range set on --service-cluster-ip-range. If/when the user decides to use dual stack services
// the Secondary* must be of different IPFamily than --service-cluster-ip-range
if secondaryServiceClusterIPRangeUsed {
// Should be dualstack IPFamily(PrimaryServiceClusterIPRange) != IPFamily(SecondaryServiceClusterIPRange)
dualstack, err := netutils.IsDualStackCIDRs([]*net.IPNet{&options.PrimaryServiceClusterIPRange, &options.SecondaryServiceClusterIPRange})
if err != nil {
errs = append(errs, errors.New("error attempting to validate dualstack for --service-cluster-ip-range and --secondary-service-cluster-ip-range"))
}

if !dualstack {
errs = append(errs, errors.New("--service-cluster-ip-range and --secondary-service-cluster-ip-range must be of different IP family"))
}

// should be smallish sized cidr, this thing is kept in etcd
// bigger cidr (specially those offered by IPv6) will add no value
// significantly increase snapshotting time.
var ones, bits = options.SecondaryServiceClusterIPRange.Mask.Size()
if bits-ones > 20 {
errs = append(errs, errors.New("specified --secondary-service-cluster-ip-range is too large"))
}
}

return errs
}

Expand Down
132 changes: 132 additions & 0 deletions cmd/kube-apiserver/app/options/validation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
"net"
"testing"

utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
)

func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *ServerRunOptions {
value := serviceCIDR
if len(secondaryServiceCIDR) > 0 {
value = value + "," + secondaryServiceCIDR
}

var primaryCIDR, secondaryCIDR net.IPNet
if len(serviceCIDR) > 0 {
_, cidr, _ := net.ParseCIDR(serviceCIDR)
if cidr != nil {
primaryCIDR = *(cidr)
}
}

if len(secondaryServiceCIDR) > 0 {
_, cidr, _ := net.ParseCIDR(secondaryServiceCIDR)
if cidr != nil {
secondaryCIDR = *(cidr)
}
}
return &ServerRunOptions{
ServiceClusterIPRanges: value,
PrimaryServiceClusterIPRange: primaryCIDR,
SecondaryServiceClusterIPRange: secondaryCIDR,
}
}

func TestClusterSerivceIPRange(t *testing.T) {
testCases := []struct {
name string
options *ServerRunOptions
enableDualStack bool
expectErrors bool
}{
{
name: "no service cidr",
expectErrors: true,
options: makeOptionsWithCIDRs("", ""),
enableDualStack: false,
},
{
name: "only secondary service cidr, dual stack gate on",
expectErrors: true,
options: makeOptionsWithCIDRs("", "10.0.0.0/16"),
enableDualStack: true,
},
{
name: "only secondary service cidr, dual stack gate off",
expectErrors: true,
options: makeOptionsWithCIDRs("", "10.0.0.0/16"),
enableDualStack: false,
},
{
name: "primary and secondary are provided but not dual stack v4-v4",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/16", "11.0.0.0/16"),
enableDualStack: true,
},
{
name: "primary and secondary are provided but not dual stack v6-v6",
expectErrors: true,
options: makeOptionsWithCIDRs("2000::/108", "3000::/108"),
enableDualStack: true,
},
{
name: "valid dual stack with gate disabled",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"),
enableDualStack: false,
},
/* success cases */
{
name: "valid primary",
expectErrors: false,
options: makeOptionsWithCIDRs("10.0.0.0/16", ""),
enableDualStack: false,
},
{
name: "valid v4-v6 dual stack + gate on",
expectErrors: false,
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"),
enableDualStack: true,
},
{
name: "valid v6-v4 dual stack + gate on",
expectErrors: false,
options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"),
enableDualStack: true,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
errs := validateClusterIPFlags(tc.options)
if len(errs) > 0 && !tc.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}

if len(errs) == 0 && tc.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}
64 changes: 57 additions & 7 deletions cmd/kube-apiserver/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -306,11 +306,21 @@ func CreateKubeAPIServerConfig(
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})

serviceIPRange, apiServerServiceIP, lastErr := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
serviceIPRange, apiServerServiceIP, lastErr := master.DefaultServiceIPRange(s.PrimaryServiceClusterIPRange)
if lastErr != nil {
return
}

// defaults to empty range and ip
var secondaryServiceIPRange net.IPNet
// process secondary range only if provided by user
if s.SecondaryServiceClusterIPRange.IP != nil {
secondaryServiceIPRange, _, lastErr = master.DefaultServiceIPRange(s.SecondaryServiceClusterIPRange)
if lastErr != nil {
return
}
}

clientCA, lastErr := readCAorNil(s.Authentication.ClientCert.ClientCA)
if lastErr != nil {
return
Expand Down Expand Up @@ -341,8 +351,10 @@ func CreateKubeAPIServerConfig(

Tunneler: nodeTunneler,

ServiceIPRange: serviceIPRange,
APIServerServiceIP: apiServerServiceIP,
ServiceIPRange: serviceIPRange,
APIServerServiceIP: apiServerServiceIP,
SecondaryServiceIPRange: secondaryServiceIPRange,

APIServerServicePort: 443,

ServiceNodePortRange: s.ServiceNodePortRange,
Expand Down Expand Up @@ -548,11 +560,49 @@ func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) {
if err := kubeoptions.DefaultAdvertiseAddress(s.GenericServerRunOptions, s.InsecureServing.DeprecatedInsecureServingOptions); err != nil {
return options, err
}
serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
if err != nil {
return options, fmt.Errorf("error determining service IP ranges: %v", err)

// process s.ServiceClusterIPRange from list to Primary and Secondary
// we process secondary only if provided by user

serviceClusterIPRangeList := strings.Split(s.ServiceClusterIPRanges, ",")

var apiServerServiceIP net.IP
var serviceIPRange net.IPNet
var err error
// nothing provided by user, use default range (only applies to the Primary)
if len(serviceClusterIPRangeList) == 0 {
var primaryServiceClusterCIDR net.IPNet
serviceIPRange, apiServerServiceIP, err = master.DefaultServiceIPRange(primaryServiceClusterCIDR)
if err != nil {
return options, fmt.Errorf("error determining service IP ranges: %v", err)
}
s.PrimaryServiceClusterIPRange = serviceIPRange
}

if len(serviceClusterIPRangeList) > 0 {
_, primaryServiceClusterCIDR, err := net.ParseCIDR(serviceClusterIPRangeList[0])
if err != nil {
return options, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr")
}

serviceIPRange, apiServerServiceIP, err = master.DefaultServiceIPRange(*(primaryServiceClusterCIDR))
if err != nil {
return options, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err)
}
s.PrimaryServiceClusterIPRange = serviceIPRange
}
s.ServiceClusterIPRange = serviceIPRange

// user provided at least two entries
if len(serviceClusterIPRangeList) > 1 {
_, secondaryServiceClusterCIDR, err := net.ParseCIDR(serviceClusterIPRangeList[1])
if err != nil {
return options, fmt.Errorf("service-cluster-ip-range[1] is not an ip net")
}

s.SecondaryServiceClusterIPRange = *(secondaryServiceClusterCIDR)
}
//note: validation asserts that the list is max of two dual stack entries

if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
return options, fmt.Errorf("error creating self-signed certificates: %v", err)
}
Expand Down
Loading

0 comments on commit 550fb1b

Please sign in to comment.