Skip to content

Commit

Permalink
feat: improve security of Kubernetes control plane components
Browse files Browse the repository at this point in the history
Fixes #3765

See #3581

There are several changes:

* `kube-controller-manager` insecure port is disabled
* `kube-controller-manager` and `kube-scheduler` now listen securely
only on localhost by default, this can be overridden with `--bind-addr`
in extra args
* `kube-controller-manager` and `kube-scheduler` now use kubeconfig with
limited access role instead of admin one

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
  • Loading branch information
smira authored and talos-bot committed Jun 18, 2021
1 parent 3aae94e commit a941eb7
Show file tree
Hide file tree
Showing 8 changed files with 223 additions and 96 deletions.
Expand Up @@ -300,6 +300,8 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
"/go-runner",
"/usr/local/bin/kube-controller-manager",
"--allocate-node-cidrs=true",
"--bind-address=127.0.0.1",
"--port=0",
fmt.Sprintf("--cluster-cidr=%s", cfg.PodCIDR),
fmt.Sprintf("--service-cluster-ip-range=%s", cfg.ServiceCIDR),
fmt.Sprintf("--cluster-signing-cert-file=%s", filepath.Join(constants.KubernetesControllerManagerSecretsDir, "ca.crt")),
Expand All @@ -310,6 +312,7 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
fmt.Sprintf("--root-ca-file=%s", filepath.Join(constants.KubernetesControllerManagerSecretsDir, "ca.crt")),
fmt.Sprintf("--service-account-private-key-file=%s", filepath.Join(constants.KubernetesControllerManagerSecretsDir, "service-account.key")),
"--profiling=false",
"--use-service-account-credentials",
}

if cfg.CloudProvider != "" {
Expand Down Expand Up @@ -356,6 +359,7 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/healthz",
Host: "localhost",
Port: intstr.FromInt(10257),
Scheme: v1.URISchemeHTTPS,
},
Expand Down Expand Up @@ -401,6 +405,7 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
"/go-runner",
"/usr/local/bin/kube-scheduler",
fmt.Sprintf("--kubeconfig=%s", filepath.Join(constants.KubernetesSchedulerSecretsDir, "kubeconfig")),
"--bind-address=127.0.0.1",
"--leader-elect=true",
"--profiling=false",
}
Expand Down Expand Up @@ -445,6 +450,7 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/healthz",
Host: "localhost",
Port: intstr.FromInt(10259),
Scheme: v1.URISchemeHTTPS,
},
Expand Down
Expand Up @@ -220,7 +220,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
templates: []template{
{
filename: "kubeconfig",
template: []byte("{{ .Secrets.AdminKubeconfig }}"),
template: []byte("{{ .Secrets.ControllerManagerKubeconfig }}"),
},
},
},
Expand All @@ -230,7 +230,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
templates: []template{
{
filename: "kubeconfig",
template: []byte("{{ .Secrets.AdminKubeconfig }}"),
template: []byte("{{ .Secrets.SchedulerKubeconfig }}"),
},
},
},
Expand Down
40 changes: 40 additions & 0 deletions internal/app/machined/pkg/controllers/secrets/kubernetes.go
Expand Up @@ -220,6 +220,46 @@ func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.RootKubernetesS

var buf bytes.Buffer

if err = kubeconfig.Generate(&kubeconfig.GenerateInput{
ClusterName: k8sRoot.Name,

CA: k8sRoot.CA,
CertificateLifetime: KubernetesCertificateValidityDuration,

CommonName: constants.KubernetesControllerManagerOrganization,
Organization: constants.KubernetesControllerManagerOrganization,

Endpoint: k8sRoot.Endpoint.String(),
Username: constants.KubernetesControllerManagerOrganization,
ContextName: "default",
}, &buf); err != nil {
return fmt.Errorf("failed to generate controller manager kubeconfig: %w", err)
}

k8sSecrets.ControllerManagerKubeconfig = buf.String()

buf.Reset()

if err = kubeconfig.Generate(&kubeconfig.GenerateInput{
ClusterName: k8sRoot.Name,

CA: k8sRoot.CA,
CertificateLifetime: KubernetesCertificateValidityDuration,

CommonName: constants.KubernetesSchedulerOrganization,
Organization: constants.KubernetesSchedulerOrganization,

Endpoint: k8sRoot.Endpoint.String(),
Username: constants.KubernetesSchedulerOrganization,
ContextName: "default",
}, &buf); err != nil {
return fmt.Errorf("failed to generate scheduler kubeconfig: %w", err)
}

k8sSecrets.SchedulerKubeconfig = buf.String()

buf.Reset()

if err = kubeconfig.GenerateAdmin(&generateAdminAdapter{k8sRoot: k8sRoot}, &buf); err != nil {
return fmt.Errorf("failed to generate admin kubeconfig: %w", err)
}
Expand Down
89 changes: 0 additions & 89 deletions internal/pkg/kubeconfig/admin.go

This file was deleted.

131 changes: 131 additions & 0 deletions internal/pkg/kubeconfig/generate.go
@@ -0,0 +1,131 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package kubeconfig

import (
"encoding/base64"
"fmt"
"io"
"net/url"
"text/template"
"time"

"github.com/talos-systems/crypto/x509"

"github.com/talos-systems/talos/pkg/machinery/config"
"github.com/talos-systems/talos/pkg/machinery/constants"
)

const kubeConfigTemplate = `apiVersion: v1
kind: Config
clusters:
- name: {{ .ClusterName }}
cluster:
server: {{ .Endpoint }}
certificate-authority-data: {{ .CACert | base64Encode }}
users:
- name: {{ .Username }}@{{ .ClusterName }}
user:
client-certificate-data: {{ .ClientCert | base64Encode }}
client-key-data: {{ .ClientKey | base64Encode }}
contexts:
- context:
cluster: {{ .ClusterName }}
namespace: default
user: {{ .Username }}@{{ .ClusterName }}
name: {{ .ContextName }}@{{ .ClusterName }}
current-context: {{ .ContextName }}@{{ .ClusterName }}
`

// GenerateAdminInput is the interface for the GenerateAdmin function.
//
// This interface is implemented by config.Cluster().
type GenerateAdminInput interface {
Name() string
Endpoint() *url.URL
CA() *x509.PEMEncodedCertificateAndKey
AdminKubeconfig() config.AdminKubeconfig
}

// GenerateAdmin generates admin kubeconfig for the cluster.
func GenerateAdmin(config GenerateAdminInput, out io.Writer) error {
return Generate(
&GenerateInput{
ClusterName: config.Name(),
CA: config.CA(),
CertificateLifetime: config.AdminKubeconfig().CertLifetime(),

CommonName: constants.KubernetesAdminCertCommonName,
Organization: constants.KubernetesAdminCertOrganization,

Endpoint: config.Endpoint().String(),
Username: "admin",
ContextName: "admin",
},
out,
)
}

// GenerateInput are input parameters for Generate.
type GenerateInput struct {
ClusterName string

CA *x509.PEMEncodedCertificateAndKey
CertificateLifetime time.Duration

CommonName string
Organization string

Endpoint string
Username string
ContextName string
}

// Generate a kubeconfig for the cluster from the given Input.
func Generate(in *GenerateInput, out io.Writer) error {
tpl, err := template.New("kubeconfig").Funcs(template.FuncMap{
"base64Encode": base64Encode,
}).Parse(kubeConfigTemplate)
if err != nil {
return fmt.Errorf("error parsing kubeconfig template: %w", err)
}

k8sCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(in.CA)
if err != nil {
return fmt.Errorf("error getting Kubernetes CA: %w", err)
}

clientCert, err := x509.NewKeyPair(k8sCA,
x509.CommonName(in.CommonName),
x509.Organization(in.Organization),
x509.NotAfter(time.Now().Add(in.CertificateLifetime)))
if err != nil {
return fmt.Errorf("error generating Kubernetes client certificate: %w", err)
}

clientCertPEM := x509.NewCertificateAndKeyFromKeyPair(clientCert)

return tpl.Execute(out, struct {
GenerateInput

CACert string
ClientCert string
ClientKey string
}{
GenerateInput: *in,
CACert: string(in.CA.Crt),
ClientCert: string(clientCertPEM.Crt),
ClientKey: string(clientCertPEM.Key),
})
}

func base64Encode(content interface{}) (string, error) {
str, ok := content.(string)
if !ok {
return "", fmt.Errorf("argument to base64 encode is not a string: %v", content)
}

return base64.StdEncoding.EncodeToString([]byte(str)), nil
}
Expand Up @@ -19,11 +19,11 @@ import (
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
)

type AdminSuite struct {
type GenerateSuite struct {
suite.Suite
}

func (suite *AdminSuite) TestGenerate() {
func (suite *GenerateSuite) TestGenerateAdmin() {
for _, rsa := range []bool{true, false} {
rsa := rsa

Expand Down Expand Up @@ -63,6 +63,37 @@ func (suite *AdminSuite) TestGenerate() {
}
}

func TestAdminSuite(t *testing.T) {
suite.Run(t, new(AdminSuite))
func (suite *GenerateSuite) TestGenerate() {
ca, err := x509.NewSelfSignedCertificateAuthority(x509.RSA(false))
suite.Require().NoError(err)

k8sCA := x509.NewCertificateAndKeyFromCertificateAuthority(ca)

input := kubeconfig.GenerateInput{
ClusterName: "foo",

CA: k8sCA,
CertificateLifetime: time.Hour,

CommonName: "system:kube-controller-manager",
Organization: "system:kube-controller-manager",

Endpoint: "https://localhost:6443/",
Username: "kube-controller-manager",
ContextName: "kube-controller-manager",
}

var buf bytes.Buffer

suite.Require().NoError(kubeconfig.Generate(&input, &buf))

// verify config via k8s client
config, err := clientcmd.Load(buf.Bytes())
suite.Require().NoError(err)

suite.Assert().NoError(clientcmd.ConfirmUsable(*config, "kube-controller-manager@foo"))
}

func TestGenerateSuite(t *testing.T) {
suite.Run(t, new(GenerateSuite))
}
6 changes: 6 additions & 0 deletions pkg/machinery/constants/constants.go
Expand Up @@ -156,6 +156,12 @@ const (
// KubernetesAdminCertOrganization defines Organization values of Kubernetes admin certificate.
KubernetesAdminCertOrganization = "system:masters"

// KubernetesControllerManagerOrganization defines Organization value of kube-controller-manager client certificate.
KubernetesControllerManagerOrganization = "system:kube-controller-manager"

// KubernetesSchedulerOrganization defines Organization value of kube-scheduler client certificate.
KubernetesSchedulerOrganization = "system:kube-scheduler"

// KubernetesAdminCertDefaultLifetime defines default lifetime for Kubernetes generated admin certificate.
KubernetesAdminCertDefaultLifetime = 365 * 24 * time.Hour

Expand Down

0 comments on commit a941eb7

Please sign in to comment.