Skip to content

Commit

Permalink
refactor(apiserver): ignore the insecure flags
Browse files Browse the repository at this point in the history 
Leave the insecure flags intact but stop serving on insecure port.
  • Loading branch information
@knight42
knight42 committed Oct 29, 2020
1 parent 5935fcd commit cfc2b33
Show file tree
Hide file tree
Showing 15 changed files with 118 additions and 71 deletions.
3 changes: 1 addition & 2 deletions cmd/kube-apiserver/app/BUILD
View file
Expand Up @@ -23,8 +23,6 @@ go_library(
"//pkg/kubeapiserver/admission:go_default_library",
"//pkg/kubeapiserver/authenticator:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/kubeapiserver/options:go_default_library",
"//pkg/kubeapiserver/server:go_default_library",
"//pkg/registry/rbac/rest:go_default_library",
"//pkg/serviceaccount:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
Expand Down Expand Up @@ -74,6 +72,7 @@ go_library(
"//staging/src/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister:go_default_library",
"//vendor/github.com/spf13/cobra:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
],
)
Expand Down
33 changes: 28 additions & 5 deletions cmd/kube-apiserver/app/options/options.go
View file
Expand Up @@ -22,12 +22,14 @@ import (
"strings"
"time"

"github.com/spf13/pflag"
utilnet "k8s.io/apimachinery/pkg/util/net"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"

api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/cluster/ports"
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
Expand All @@ -37,12 +39,15 @@ import (
"k8s.io/kubernetes/pkg/serviceaccount"
)

// InsecurePortFlags are dummy flags, they are kept only for compatibility and will be removed in v1.24.
// TODO: remove these flags in v1.24.
var InsecurePortFlags = []string{"insecure-port", "port"}

// ServerRunOptions runs a kubernetes api server.
type ServerRunOptions struct {
GenericServerRunOptions *genericoptions.ServerRunOptions
Etcd *genericoptions.EtcdOptions
SecureServing *genericoptions.SecureServingOptionsWithLoopback
InsecureServing *genericoptions.DeprecatedInsecureServingOptionsWithLoopback
Audit *genericoptions.AuditOptions
Features *genericoptions.FeatureOptions
Admission *kubeoptions.AdmissionOptions
Expand All @@ -62,7 +67,7 @@ type ServerRunOptions struct {
MaxConnectionBytesPerSec int64
// ServiceClusterIPRange is mapped to input provided by user
ServiceClusterIPRanges string
//PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results
// PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results
// of parsing ServiceClusterIPRange into actual values
PrimaryServiceClusterIPRange net.IPNet
SecondaryServiceClusterIPRange net.IPNet
Expand Down Expand Up @@ -92,7 +97,6 @@ func NewServerRunOptions() *ServerRunOptions {
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
SecureServing: kubeoptions.NewSecureServingOptions(),
InsecureServing: kubeoptions.NewInsecureServingOptions(),
Audit: genericoptions.NewAuditOptions(),
Features: genericoptions.NewFeatureOptions(),
Admission: kubeoptions.NewAdmissionOptions(),
Expand Down Expand Up @@ -134,14 +138,33 @@ func NewServerRunOptions() *ServerRunOptions {
return &s
}

// TODO: remove these insecure flags in v1.24
func addDummyInsecureFlags(fs *pflag.FlagSet) {
var (
bindAddr = net.IPv4(127, 0, 0, 1)
bindPort int
)

for _, name := range []string{"insecure-bind-address", "address"} {
fs.IPVar(&bindAddr, name, bindAddr, ""+
"The IP address on which to serve the insecure port (set to 0.0.0.0 for all IPv4 interfaces and :: for all IPv6 interfaces).")
fs.MarkDeprecated(name, "This flag has no effect now and will be removed in v1.24.")
}

for _, name := range InsecurePortFlags {
fs.IntVar(&bindPort, name, bindPort, ""+
"The port on which to serve unsecured, unauthenticated access.")
fs.MarkDeprecated(name, "This flag has no effect now and will be removed in v1.24.")
}
}

// Flags returns flags for a specific APIServer by section name
func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
// Add the generic flags.
s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
s.Etcd.AddFlags(fss.FlagSet("etcd"))
s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
s.InsecureServing.AddFlags(fss.FlagSet("insecure serving"))
s.InsecureServing.AddUnqualifiedFlags(fss.FlagSet("insecure serving")) // TODO: remove it until kops stops using `--address`
addDummyInsecureFlags(fss.FlagSet("insecure serving"))
s.Audit.AddFlags(fss.FlagSet("auditing"))
s.Features.AddFlags(fss.FlagSet("features"))
s.Authentication.AddFlags(fss.FlagSet("authentication"))
Expand Down
4 changes: 0 additions & 4 deletions cmd/kube-apiserver/app/options/options_test.go
View file
Expand Up @@ -177,10 +177,6 @@ func TestAddFlags(t *testing.T) {
HTTP2MaxStreamsPerConnection: 42,
Required: true,
}).WithLoopback(),
InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
BindAddress: net.ParseIP("127.0.0.1"),
BindPort: 8080,
}).WithLoopback(),
EventTTL: 1 * time.Hour,
KubeletConfig: kubeletclient.KubeletClientConfig{
Port: 10250,
Expand Down
1 change: 0 additions & 1 deletion cmd/kube-apiserver/app/options/validation.go
View file
Expand Up @@ -170,7 +170,6 @@ func (s *ServerRunOptions) Validate() []error {
errs = append(errs, s.Authorization.Validate()...)
errs = append(errs, s.Audit.Validate()...)
errs = append(errs, s.Admission.Validate()...)
errs = append(errs, s.InsecureServing.Validate()...)
errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...)
errs = append(errs, validateTokenRequest(s)...)
errs = append(errs, s.Metrics.Validate()...)
Expand Down
64 changes: 34 additions & 30 deletions cmd/kube-apiserver/app/server.go
View file
Expand Up @@ -31,6 +31,7 @@ import (
"time"

"github.com/spf13/cobra"
"github.com/spf13/pflag"

extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
Expand Down Expand Up @@ -65,6 +66,7 @@ import (
"k8s.io/klog/v2"
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"

"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/capabilities"
Expand All @@ -77,8 +79,6 @@ import (
kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
kubeserver "k8s.io/kubernetes/pkg/kubeapiserver/server"
rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
"k8s.io/kubernetes/pkg/serviceaccount"
)
Expand All @@ -88,6 +88,20 @@ const (
etcdRetryInterval = 1 * time.Second
)

// TODO: delete this check after insecure flags removed in v1.24
func checkNonZeroInsecurePort(fs *pflag.FlagSet) error {
for _, name := range options.InsecurePortFlags {
val, err := fs.GetInt(name)
if err != nil {
return err
}
if val != 0 {
return fmt.Errorf("invalid port value %d: only zero is allowed", val)
}
}
return nil
}

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
s := options.NewServerRunOptions()
Expand All @@ -108,8 +122,13 @@ cluster's shared state through which all other components interact.`,
},
RunE: func(cmd *cobra.Command, args []string) error {
verflag.PrintAndExitIfRequested()
cliflag.PrintFlags(cmd.Flags())
fs := cmd.Flags()
cliflag.PrintFlags(fs)

err := checkNonZeroInsecurePort(fs)
if err != nil {
return err
}
// set default options
completedOptions, err := Complete(s)
if err != nil {
Expand Down Expand Up @@ -182,7 +201,7 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan
return nil, err
}

kubeAPIServerConfig, insecureServingInfo, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions, nodeTunneler, proxyTransport)
kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions, nodeTunneler, proxyTransport)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -214,13 +233,6 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan
return nil, err
}

if insecureServingInfo != nil {
insecureHandlerChain := kubeserver.BuildInsecureHandlerChain(aggregatorServer.GenericAPIServer.UnprotectedHandler(), kubeAPIServerConfig.GenericConfig)
if err := insecureServingInfo.Serve(insecureHandlerChain, kubeAPIServerConfig.GenericConfig.RequestTimeout, stopCh); err != nil {
return nil, err
}
}

return aggregatorServer, nil
}

Expand Down Expand Up @@ -288,19 +300,18 @@ func CreateKubeAPIServerConfig(
proxyTransport *http.Transport,
) (
*controlplane.Config,
*genericapiserver.DeprecatedInsecureServingInfo,
aggregatorapiserver.ServiceResolver,
[]admission.PluginInitializer,
error,
) {
genericConfig, versionedInformers, insecureServingInfo, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport)
genericConfig, versionedInformers, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport)
if err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}

if _, port, err := net.SplitHostPort(s.Etcd.StorageConfig.Transport.ServerList[0]); err == nil && port != "0" && len(port) != 0 {
if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.Transport.ServerList}.CheckEtcdServers); err != nil {
return nil, nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err)
return nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err)
}
}

Expand All @@ -322,7 +333,7 @@ func CreateKubeAPIServerConfig(

serviceIPRange, apiServerServiceIP, err := controlplane.ServiceIPRange(s.PrimaryServiceClusterIPRange)
if err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}

// defaults to empty range and ip
Expand All @@ -331,7 +342,7 @@ func CreateKubeAPIServerConfig(
if s.SecondaryServiceClusterIPRange.IP != nil {
secondaryServiceIPRange, _, err = controlplane.ServiceIPRange(s.SecondaryServiceClusterIPRange)
if err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}
}

Expand Down Expand Up @@ -369,13 +380,13 @@ func CreateKubeAPIServerConfig(

clientCAProvider, err := s.Authentication.ClientCert.GetClientCAContentProvider()
if err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}
config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider

requestHeaderConfig, err := s.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig()
if err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}
if requestHeaderConfig != nil {
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider
Expand All @@ -386,7 +397,7 @@ func CreateKubeAPIServerConfig(
}

if err := config.GenericConfig.AddPostStartHook("start-kube-apiserver-admission-initializer", admissionPostStartHook); err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}

if nodeTunneler != nil {
Expand All @@ -401,7 +412,7 @@ func CreateKubeAPIServerConfig(
networkContext := egressselector.Cluster.AsNetworkContext()
dialer, err := config.GenericConfig.EgressSelector.Lookup(networkContext)
if err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}
c := proxyTransport.Clone()
c.DialContext = dialer
Expand All @@ -414,7 +425,7 @@ func CreateKubeAPIServerConfig(
for _, f := range s.Authentication.ServiceAccounts.KeyFiles {
keys, err := keyutil.PublicKeysFromFile(f)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err)
return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err)
}
pubKeys = append(pubKeys, keys...)
}
Expand All @@ -424,7 +435,7 @@ func CreateKubeAPIServerConfig(
config.ExtraConfig.ServiceAccountPublicKeys = pubKeys
}

return config, insecureServingInfo, serviceResolver, pluginInitializers, nil
return config, serviceResolver, pluginInitializers, nil
}

// BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it
Expand All @@ -434,7 +445,6 @@ func buildGenericConfig(
) (
genericConfig *genericapiserver.Config,
versionedInformers clientgoinformers.SharedInformerFactory,
insecureServingInfo *genericapiserver.DeprecatedInsecureServingInfo,
serviceResolver aggregatorapiserver.ServiceResolver,
pluginInitializers []admission.PluginInitializer,
admissionPostStartHook genericapiserver.PostStartHookFunc,
Expand All @@ -448,9 +458,6 @@ func buildGenericConfig(
return
}

if lastErr = s.InsecureServing.ApplyTo(&insecureServingInfo, &genericConfig.LoopbackClientConfig); lastErr != nil {
return
}
if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil {
return
}
Expand Down Expand Up @@ -595,9 +602,6 @@ func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) {
if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing.SecureServingOptions); err != nil {
return options, err
}
if err := kubeoptions.DefaultAdvertiseAddress(s.GenericServerRunOptions, s.InsecureServing.DeprecatedInsecureServingOptions); err != nil {
return options, err
}

// process s.ServiceClusterIPRange from list to Primary and Secondary
// we process secondary only if provided by user
Expand Down
1 change: 0 additions & 1 deletion cmd/kube-apiserver/app/testing/testserver.go
View file
Expand Up @@ -122,7 +122,6 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
s.InsecureServing.BindPort = 0

s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
if err != nil {
Expand Down
16 changes: 15 additions & 1 deletion hack/lib/util.sh
View file
Expand Up @@ -45,6 +45,7 @@ kube::util::wait_for_url() {
local wait=${3:-1}
local times=${4:-30}
local maxtime=${5:-1}
local extra_args=${6:-}

command -v curl >/dev/null || {
kube::log::usage "curl must be installed"
Expand All @@ -54,7 +55,9 @@ kube::util::wait_for_url() {
local i
for i in $(seq 1 "${times}"); do
local out
if out=$(curl --max-time "${maxtime}" -gkfs "${url}" 2>/dev/null); then
# shellcheck disable=SC2086
# Disabling because "${extra_args}" needs to allow for expansion here
if out=$(curl --max-time "${maxtime}" -gkfs $extra_args "${url}" 2>/dev/null); then
kube::log::status "On try ${i}, ${prefix}: ${out}"
return 0
fi
Expand All @@ -64,6 +67,17 @@ kube::util::wait_for_url() {
return 1
}

kube::util::wait_for_url_with_bearer_token() {
local url=$1
local token=$2
local prefix=${3:-}
local wait=${4:-1}
local times=${5:-30}
local maxtime=${6:-1}

kube::util::wait_for_url "${url}" "${prefix}" "${wait}" "${times}" "${maxtime}" "--oauth2-bearer ${token}"
}

# Example: kube::util::wait_for_success 120 5 "kubectl get nodes|grep localhost"
# arguments: wait time, sleep time, shell command
# returns 0 if the shell command get output, 1 otherwise.
Expand Down

0 comments on commit cfc2b33

Please sign in to comment.