From 812a0ff1a2ed25f57f67fc0b6eedd919f3599373 Mon Sep 17 00:00:00 2001 From: Axel Christ Date: Tue, 14 Mar 2023 11:51:52 +0100 Subject: [PATCH] Implement apinet apinetlet bootstrap * Add apinetlet bootstrapper role, group and actual role and group * Add approval controller for approving apinet apinetlet certificate requests * Add tests --- api/v1alpha1/common_types.go | 11 ++ apinetlet/client/config/getter.go | 34 ++++- apinetlet/main.go | 17 ++- .../rbac/apinetlet_bootstrapper_role.yaml | 20 +++ .../apinetlet_bootstrapper_rolebinding.yaml | 14 ++ .../onmetal-api-net/rbac/apinetlet_role.yaml | 31 +++++ .../rbac/apinetlet_rolebinding.yaml | 12 ++ .../onmetal-api-net/rbac/kustomization.yaml | 5 + config/onmetal-api-net/rbac/role.yaml | 30 ++++ go.mod | 1 + go.sum | 7 + .../certificate/generic/certificate.go | 95 +++++++++++++ .../certificate/onmetal-api-net/apinetlet.go | 90 ++++++++++++ .../onmetal-api-net/onmetalapinet.go | 19 +++ .../certificateapproval_controller.go | 129 ++++++++++++++++++ .../controllers/certificateapproval_test.go | 62 +++++++++ .../controllers/controllers_suite_test.go | 80 +++++------ .../controllers/network_controller_test.go | 8 +- .../controllers/publicip_controller_test.go | 8 +- onmetal-api-net/main.go | 9 ++ 20 files changed, 625 insertions(+), 57 deletions(-) create mode 100644 config/onmetal-api-net/rbac/apinetlet_bootstrapper_role.yaml create mode 100644 config/onmetal-api-net/rbac/apinetlet_bootstrapper_rolebinding.yaml create mode 100644 config/onmetal-api-net/rbac/apinetlet_role.yaml create mode 100644 config/onmetal-api-net/rbac/apinetlet_rolebinding.yaml create mode 100644 onmetal-api-net/controllers/certificate/generic/certificate.go create mode 100644 onmetal-api-net/controllers/certificate/onmetal-api-net/apinetlet.go create mode 100644 onmetal-api-net/controllers/certificate/onmetal-api-net/onmetalapinet.go create mode 100644 onmetal-api-net/controllers/certificateapproval_controller.go create mode 100644 onmetal-api-net/controllers/certificateapproval_test.go diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index faf99b8d..6b78541d 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -24,8 +24,19 @@ import ( const ( ReconcileRequestAnnotation = "reconcile.apinet.api.onmetal.de/requestedAt" + + // APINetletsGroup is the system rbac group all apinetlets are in. + APINetletsGroup = "apinet.api.onmetal.de:system:apinetlets" + + // APINetletUserNamePrefix is the prefix all apinetlet users should have. + APINetletUserNamePrefix = "apinet.api.onmetal.de:system:apinetlet:" ) +// APINetletCommonName constructs the common name for a certificate of an apinetlet user. +func APINetletCommonName(name string) string { + return APINetletUserNamePrefix + name +} + // IP is an IP address. // +kubebuilder:validation:Type=string type IP struct { diff --git a/apinetlet/client/config/getter.go b/apinetlet/client/config/getter.go index d8419d5b..be3916e2 100644 --- a/apinetlet/client/config/getter.go +++ b/apinetlet/client/config/getter.go @@ -17,16 +17,21 @@ package config import ( "crypto/x509" "crypto/x509/pkix" + "os" + onmetalapinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/v1alpha1" networkingv1alpha1 "github.com/onmetal/onmetal-api/api/networking/v1alpha1" utilcertificate "github.com/onmetal/onmetal-api/utils/certificate" "github.com/onmetal/onmetal-api/utils/client/config" certificatesv1 "k8s.io/api/certificates/v1" "k8s.io/apiserver/pkg/server/egressselector" + ctrl "sigs.k8s.io/controller-runtime" ) +var log = ctrl.Log.WithName("client").WithName("config") + var ( - APINetGetter = config.NewGetterOrDie(config.GetterOptions{ + Getter = config.NewGetterOrDie(config.GetterOptions{ Name: "apinetlet", SignerName: certificatesv1.KubeAPIServerClientSignerName, Template: &x509.CertificateRequest{ @@ -39,5 +44,30 @@ var ( NetworkContext: egressselector.ControlPlane.AsNetworkContext(), }) - APINetGetConfig = APINetGetter.GetConfig + GetConfig = Getter.GetConfig + GetConfigOrDie = Getter.GetConfigOrDie ) + +func NewAPINetGetter(namespace string) (*config.Getter, error) { + return config.NewGetter(config.GetterOptions{ + Name: "apinetlet", + SignerName: certificatesv1.KubeAPIServerClientSignerName, + Template: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: onmetalapinetv1alpha1.APINetletCommonName(namespace), + Organization: []string{onmetalapinetv1alpha1.APINetletsGroup}, + }, + }, + GetUsages: utilcertificate.DefaultKubeAPIServerClientGetUsages, + NetworkContext: egressselector.ControlPlane.AsNetworkContext(), + }) +} + +func NewAPINetGetterOrDie(namespace string) *config.Getter { + getter, err := NewAPINetGetter(namespace) + if err != nil { + log.Error(err, "Error creating getter") + os.Exit(1) + } + return getter +} diff --git a/apinetlet/main.go b/apinetlet/main.go index 7405bc62..9813ddb4 100644 --- a/apinetlet/main.go +++ b/apinetlet/main.go @@ -22,7 +22,6 @@ import ( "fmt" "os" - "github.com/onmetal/controller-utils/configutils" onmetalapinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/v1alpha1" apinetletconfig "github.com/onmetal/onmetal-api-net/apinetlet/client/config" "github.com/onmetal/onmetal-api-net/apinetlet/controllers" @@ -68,6 +67,7 @@ func main() { var enableLeaderElection bool var probeAddr string + var configOptions config.GetConfigOptions var apiNetGetConfigOptions config.GetConfigOptions var apiNetNamespace string @@ -81,6 +81,7 @@ func main() { "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + configOptions.BindFlags(flag.CommandLine) apiNetGetConfigOptions.BindFlags(flag.CommandLine, config.WithNamePrefix(apiNetFlagPrefix)) flag.StringVar(&apiNetNamespace, "api-net-namespace", "", "api-net cluster namespace to manage all objects in.") @@ -91,8 +92,9 @@ func main() { opts := zap.Options{ Development: true, } - opts.BindFlags(goflag.CommandLine) - flag.CommandLine.AddGoFlagSet(goflag.CommandLine) + goFlags := goflag.NewFlagSet(os.Args[0], goflag.ExitOnError) + opts.BindFlags(goFlags) + flag.CommandLine.AddGoFlagSet(goFlags) flag.Parse() ctx := ctrl.SetupSignalHandler() @@ -108,13 +110,14 @@ func main() { setupLog.Info("Watching onmetal-api objects only in namespace for reconciliation", "namespace", watchNamespace) } - cfg, err := configutils.GetConfig() + cfg, cfgCtrl, err := apinetletconfig.GetConfig(ctx, &configOptions) if err != nil { setupLog.Error(err, "unable to load kubeconfig") os.Exit(1) } - apiNetCfg, apiNetCfgCtrl, err := apinetletconfig.APINetGetConfig(ctx, &apiNetGetConfigOptions) + apiNetGetter := apinetletconfig.NewAPINetGetterOrDie(apiNetNamespace) + apiNetCfg, apiNetCfgCtrl, err := apiNetGetter.GetConfig(ctx, &apiNetGetConfigOptions) if err != nil { setupLog.Error(err, "unable to load api net kubeconfig") os.Exit(1) @@ -133,6 +136,10 @@ func main() { setupLog.Error(err, "unable to start manager") os.Exit(1) } + if err := config.SetupControllerWithManager(mgr, cfgCtrl); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Config") + os.Exit(1) + } if err := config.SetupControllerWithManager(mgr, apiNetCfgCtrl); err != nil { setupLog.Error(err, "unable to create controller", "controller", "APINetConfig") os.Exit(1) diff --git a/config/onmetal-api-net/rbac/apinetlet_bootstrapper_role.yaml b/config/onmetal-api-net/rbac/apinetlet_bootstrapper_role.yaml new file mode 100644 index 00000000..46bda8f2 --- /dev/null +++ b/config/onmetal-api-net/rbac/apinetlet_bootstrapper_role.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: apinet.api.onmetal.de:system:apinetlet-bootstrapper +rules: + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - create + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/apinetletclient + verbs: + - create \ No newline at end of file diff --git a/config/onmetal-api-net/rbac/apinetlet_bootstrapper_rolebinding.yaml b/config/onmetal-api-net/rbac/apinetlet_bootstrapper_rolebinding.yaml new file mode 100644 index 00000000..53cd1e5d --- /dev/null +++ b/config/onmetal-api-net/rbac/apinetlet_bootstrapper_rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: apinet.api.onmetal.de:system:apinetlet-bootstrapper +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: apinet.api.onmetal.de:system:apinetlet-bootstrapper +subjects: + - kind: Group + # Group name has to match bootstrap group pattern \Asystem:bootstrappers:[a-z0-9:-]{0,255}[a-z0-9]\ + # See https://github.com/kubernetes/kubernetes/blob/e8662a46dd27db774ec953dae15f93ae2d1a68c8/staging/src/k8s.io/cluster-bootstrap/token/api/types.go#L96 + name: system:bootstrappers:apinet-api-onmetal-de:apinetlets + apiGroup: rbac.authorization.k8s.io diff --git a/config/onmetal-api-net/rbac/apinetlet_role.yaml b/config/onmetal-api-net/rbac/apinetlet_role.yaml new file mode 100644 index 00000000..489bd9d4 --- /dev/null +++ b/config/onmetal-api-net/rbac/apinetlet_role.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: apinet.api.onmetal.de:system:apinetlets +rules: +- apiGroups: + - apinet.api.onmetal.de + resources: + - networks + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - apinet.api.onmetal.de + resources: + - publicips + verbs: + - get + - list + - patch + - update + - watch + - create + - delete diff --git a/config/onmetal-api-net/rbac/apinetlet_rolebinding.yaml b/config/onmetal-api-net/rbac/apinetlet_rolebinding.yaml new file mode 100644 index 00000000..e30a0558 --- /dev/null +++ b/config/onmetal-api-net/rbac/apinetlet_rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: apinet.api.onmetal.de:system:apinetlets +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: apinet.api.onmetal.de:system:apinetlets +subjects: + - kind: Group + name: apinet.api.onmetal.de:system:apinetlets + apiGroup: rbac.authorization.k8s.io diff --git a/config/onmetal-api-net/rbac/kustomization.yaml b/config/onmetal-api-net/rbac/kustomization.yaml index 731832a6..a2ee323d 100644 --- a/config/onmetal-api-net/rbac/kustomization.yaml +++ b/config/onmetal-api-net/rbac/kustomization.yaml @@ -16,3 +16,8 @@ resources: - auth_proxy_role.yaml - auth_proxy_role_binding.yaml - auth_proxy_client_clusterrole.yaml +# APINetlet (bootstrapper) roles +- apinetlet_role.yaml +- apinetlet_rolebinding.yaml +- apinetlet_bootstrapper_role.yaml +- apinetlet_bootstrapper_rolebinding.yaml diff --git a/config/onmetal-api-net/rbac/role.yaml b/config/onmetal-api-net/rbac/role.yaml index 1943633d..3a85f9a4 100644 --- a/config/onmetal-api-net/rbac/role.yaml +++ b/config/onmetal-api-net/rbac/role.yaml @@ -60,3 +60,33 @@ rules: - get - patch - update +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resourceNames: + - kubernetes.io/kube-apiserver-client + resources: + - signers + verbs: + - approve diff --git a/go.mod b/go.mod index ab2641cd..d1dcf4b0 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/onsi/gomega v1.27.3 github.com/spf13/pflag v1.0.5 go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d + golang.org/x/exp v0.0.0-20221212164502-fae10dda9338 k8s.io/api v0.26.2 k8s.io/apimachinery v0.26.2 k8s.io/apiserver v0.26.2 diff --git a/go.sum b/go.sum index 2e50ac47..d86789a8 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= @@ -251,6 +252,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/onmetal/controller-utils v0.7.0 h1:EHLPb/XimNas1VkeZZLP4g31aSz+ipiwvwWhklaQob0= github.com/onmetal/controller-utils v0.7.0/go.mod h1:91KV/s0VaB8PC+hqsxo6OBsAhi3ICFgIFLv/36V0ng8= github.com/onmetal/onmetal-api v0.0.13-0.20230313112836-dfd3ad84912f h1:2kA63fXBwoFEjzF9JmhzxMbjX511JD3/0p5neOOLhnQ= @@ -366,6 +368,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -376,6 +379,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20221212164502-fae10dda9338 h1:OvjRkcNHnf6/W5FZXSxODbxwD+X7fspczG7Jn/xQVD4= +golang.org/x/exp v0.0.0-20221212164502-fae10dda9338/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -666,6 +671,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -701,6 +707,7 @@ k8s.io/component-base v0.26.2 h1:IfWgCGUDzrD6wLLgXEstJKYZKAFS2kO+rBRi0p3LqcI= k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kms v0.26.2 h1:GM1gg3tFK3OUU/QQFi93yGjG3lJT8s8l3Wkn2+VxBLM= k8s.io/kube-aggregator v0.26.2 h1:WtcLGisa5aCKBbBI1/Xe7gdjPlVb5Xhvs4a8Rdk8EXs= k8s.io/kube-aggregator v0.26.2/go.mod h1:swDTw0k/XghVLR+PCWnP6Y36wR2+DsqL2HUVq8eu0RI= k8s.io/kube-openapi v0.0.0-20230109183929-3758b55a6596 h1:8cNCQs+WqqnSpZ7y0LMQPKD+RZUHU17VqLPMW3qxnxc= diff --git a/onmetal-api-net/controllers/certificate/generic/certificate.go b/onmetal-api-net/controllers/certificate/generic/certificate.go new file mode 100644 index 00000000..83c6436f --- /dev/null +++ b/onmetal-api-net/controllers/certificate/generic/certificate.go @@ -0,0 +1,95 @@ +// Copyright 2023 OnMetal authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generic + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + + authv1 "k8s.io/api/authorization/v1" + certificatesv1 "k8s.io/api/certificates/v1" +) + +type CertificateSigningRequestRecognizer interface { + Recognize(csr *certificatesv1.CertificateSigningRequest, x509CR *x509.CertificateRequest) bool + Permission() authv1.ResourceAttributes + SuccessMessage() string +} + +type certificateSigningRequestRecognizer struct { + recognize func(csr *certificatesv1.CertificateSigningRequest, x509CR *x509.CertificateRequest) bool + permission authv1.ResourceAttributes + successMessage string +} + +func NewCertificateSigningRequestRecognizer( + recognize func(csr *certificatesv1.CertificateSigningRequest, x509CR *x509.CertificateRequest) bool, + permission authv1.ResourceAttributes, + successMessage string, +) CertificateSigningRequestRecognizer { + return &certificateSigningRequestRecognizer{ + recognize: recognize, + permission: permission, + successMessage: successMessage, + } +} + +func (r *certificateSigningRequestRecognizer) Recognize(csr *certificatesv1.CertificateSigningRequest, x509CR *x509.CertificateRequest) bool { + return r.recognize(csr, x509CR) +} + +func (r *certificateSigningRequestRecognizer) Permission() authv1.ResourceAttributes { + return r.permission +} + +func (r *certificateSigningRequestRecognizer) SuccessMessage() string { + return r.successMessage +} + +const ( + CertificateRequestPEMBlockType = "CERTIFICATE REQUEST" +) + +func ParseCertificateRequest(pemBytes []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != CertificateRequestPEMBlockType { + return nil, fmt.Errorf("pem block type must be %s", CertificateRequestPEMBlockType) + } + + csr, err := x509.ParseCertificateRequest(block.Bytes) + if err != nil { + return nil, err + } + + return csr, nil +} + +func GetCertificateSigningRequestApprovalCondition(status *certificatesv1.CertificateSigningRequestStatus) (approved, denied bool) { + for _, c := range status.Conditions { + if c.Type == certificatesv1.CertificateApproved { + approved = true + } + if c.Type == certificatesv1.CertificateDenied { + denied = true + } + } + return approved, denied +} + +func IsCertificateSigningRequestApproved(csr *certificatesv1.CertificateSigningRequest) bool { + approved, denied := GetCertificateSigningRequestApprovalCondition(&csr.Status) + return approved && !denied +} diff --git a/onmetal-api-net/controllers/certificate/onmetal-api-net/apinetlet.go b/onmetal-api-net/controllers/certificate/onmetal-api-net/apinetlet.go new file mode 100644 index 00000000..0ad44f91 --- /dev/null +++ b/onmetal-api-net/controllers/certificate/onmetal-api-net/apinetlet.go @@ -0,0 +1,90 @@ +// Copyright 2023 OnMetal authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package onmetalapinet + +import ( + "crypto/x509" + "fmt" + "strings" + + onmetalapinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/v1alpha1" + "github.com/onmetal/onmetal-api-net/onmetal-api-net/controllers/certificate/generic" + "golang.org/x/exp/slices" + authv1 "k8s.io/api/authorization/v1" + certificatesv1 "k8s.io/api/certificates/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +var ( + APINetletRequiredUsages = sets.New[certificatesv1.KeyUsage]( + certificatesv1.UsageDigitalSignature, + certificatesv1.UsageKeyEncipherment, + certificatesv1.UsageClientAuth, + ) +) + +func IsAPINetletClientCert(csr *certificatesv1.CertificateSigningRequest, x509cr *x509.CertificateRequest) bool { + if csr.Spec.SignerName != certificatesv1.KubeAPIServerClientSignerName { + return false + } + + return ValidateAPINetletClientCSR(x509cr, sets.New(csr.Spec.Usages...)) == nil +} + +func ValidateAPINetletClientCSR(req *x509.CertificateRequest, usages sets.Set[certificatesv1.KeyUsage]) error { + if !slices.Equal([]string{onmetalapinetv1alpha1.APINetletsGroup}, req.Subject.Organization) { + return fmt.Errorf("organization is not %s", onmetalapinetv1alpha1.APINetletsGroup) + } + + if len(req.DNSNames) > 0 { + return fmt.Errorf("dns subject alternative names are not allowed") + } + if len(req.EmailAddresses) > 0 { + return fmt.Errorf("email subject alternative names are not allowed") + } + if len(req.IPAddresses) > 0 { + return fmt.Errorf("ip subject alternative names are not allowed") + } + if len(req.URIs) > 0 { + return fmt.Errorf("uri subject alternative names are not allowed") + } + + if !strings.HasPrefix(req.Subject.CommonName, onmetalapinetv1alpha1.APINetletUserNamePrefix) { + return fmt.Errorf("subject common name does not begin with %s", onmetalapinetv1alpha1.APINetletUserNamePrefix) + } + + if !APINetletRequiredUsages.Equal(usages) { + return fmt.Errorf("usages did not match %v", sets.List(APINetletRequiredUsages)) + } + + return nil +} + +var ( + APINetletRecognizer = generic.NewCertificateSigningRequestRecognizer( + IsAPINetletClientCert, + authv1.ResourceAttributes{ + Group: certificatesv1.GroupName, + Resource: "certificatesigningrequests", + Verb: "create", + Subresource: "apinetletclient", + }, + "Auto approving apinetlet client certificate after SubjectAccessReview.", + ) +) + +func init() { + Recognizers = append(Recognizers, APINetletRecognizer) +} diff --git a/onmetal-api-net/controllers/certificate/onmetal-api-net/onmetalapinet.go b/onmetal-api-net/controllers/certificate/onmetal-api-net/onmetalapinet.go new file mode 100644 index 00000000..5532ac69 --- /dev/null +++ b/onmetal-api-net/controllers/certificate/onmetal-api-net/onmetalapinet.go @@ -0,0 +1,19 @@ +// Copyright 2023 OnMetal authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package onmetalapinet + +import "github.com/onmetal/onmetal-api-net/onmetal-api-net/controllers/certificate/generic" + +var Recognizers []generic.CertificateSigningRequestRecognizer diff --git a/onmetal-api-net/controllers/certificateapproval_controller.go b/onmetal-api-net/controllers/certificateapproval_controller.go new file mode 100644 index 00000000..e3bee694 --- /dev/null +++ b/onmetal-api-net/controllers/certificateapproval_controller.go @@ -0,0 +1,129 @@ +// Copyright 2023 OnMetal authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "fmt" + + "github.com/onmetal/onmetal-api-net/onmetal-api-net/controllers/certificate/generic" + authv1 "k8s.io/api/authorization/v1" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type CertificateApprovalReconciler struct { + client.Client + + Recognizers []generic.CertificateSigningRequestRecognizer +} + +//+kubebuilder:rbac:groups=certificates.k8s.io,resources=signers,resourceNames=kubernetes.io/kube-apiserver-client,verbs=approve +//+kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=get;list;watch +//+kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/approval,verbs=get;update;patch +//+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + +func (r *CertificateApprovalReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + csr := &certificatesv1.CertificateSigningRequest{} + if err := r.Get(ctx, req.NamespacedName, csr); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if len(csr.Status.Certificate) > 0 { + log.V(1).Info("Certificate already present, nothing to do") + return ctrl.Result{}, nil + } + + if approved, denied := generic.GetCertificateSigningRequestApprovalCondition(&csr.Status); approved || denied { + log.V(1).Info("Certificate approval already marked", "Approved", approved, "Denied", denied) + return ctrl.Result{}, nil + } + + x509CR, err := generic.ParseCertificateRequest(csr.Spec.Request) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error parsing csr: %w", err) + } + + var tried []string + for _, recognizer := range r.Recognizers { + if recognizer.Recognize(csr, x509CR) { + continue + } + + permission := recognizer.Permission() + tried = append(tried, permission.Subresource) + + approved, err := r.authorize(ctx, csr, permission) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error approving permission subresource %s: %w", + permission.Subresource, + err, + ) + } + if approved { + appendApprovalCondition(csr, recognizer.SuccessMessage()) + if err := r.Client.SubResource("approval").Update(ctx, csr); err != nil { + return ctrl.Result{}, fmt.Errorf("error updating approval for certificate signing request: %w", err) + } + + log.V(1).Info("Approved certificate", "Subresource", permission.Subresource) + return ctrl.Result{}, nil + } + } + if len(tried) > 0 { + log.V(1).Info("Recognized certificate signing request but access review was not approved", "Tried", tried) + } + + return ctrl.Result{}, nil +} + +func (r *CertificateApprovalReconciler) authorize(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, attrs authv1.ResourceAttributes) (bool, error) { + extra := make(map[string]authv1.ExtraValue) + for k, v := range csr.Spec.Extra { + extra[k] = authv1.ExtraValue(v) + } + + sar := &authv1.SubjectAccessReview{ + Spec: authv1.SubjectAccessReviewSpec{ + User: csr.Spec.Username, + UID: csr.Spec.UID, + Groups: csr.Spec.Groups, + Extra: extra, + ResourceAttributes: &attrs, + }, + } + if err := r.Client.Create(ctx, sar); err != nil { + return false, fmt.Errorf("error creating subject access review: %w", err) + } + return sar.Status.Allowed, nil +} + +func (r *CertificateApprovalReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&certificatesv1.CertificateSigningRequest{}). + Complete(r) +} + +func appendApprovalCondition(csr *certificatesv1.CertificateSigningRequest, message string) { + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + Reason: "AutoApproved", + Message: message, + }) +} diff --git a/onmetal-api-net/controllers/certificateapproval_test.go b/onmetal-api-net/controllers/certificateapproval_test.go new file mode 100644 index 00000000..6b4a20cf --- /dev/null +++ b/onmetal-api-net/controllers/certificateapproval_test.go @@ -0,0 +1,62 @@ +// Copyright 2023 OnMetal authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "crypto/x509" + "crypto/x509/pkix" + + onmetalapinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/v1alpha1" + utilcertificate "github.com/onmetal/onmetal-api/utils/certificate" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +var _ = Describe("CertificateApprovalController", func() { + DescribeTable("certificate approval", + func(ctx SpecContext, commonName, organization string) { + By("creating a certificate signing request") + csr, _, _, err := utilcertificate.GenerateAndCreateCertificateSigningRequest( + ctx, + k8sClient, + certificatesv1.KubeAPIServerClientSignerName, + &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: commonName, + Organization: []string{organization}, + }, + }, + utilcertificate.DefaultKubeAPIServerClientGetUsages, + nil, + ) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for the csr to be approved and a certificate to be available") + Eventually(ctx, Object(csr)).Should( + HaveField("Status.Conditions", ContainElement(SatisfyAll( + HaveField("Type", certificatesv1.CertificateApproved), + HaveField("Status", corev1.ConditionTrue), + ))), + ) + }, + Entry("apinetlet", + onmetalapinetv1alpha1.APINetletCommonName("my-apinetlet"), + onmetalapinetv1alpha1.APINetletsGroup, + ), + ) +}) diff --git a/onmetal-api-net/controllers/controllers_suite_test.go b/onmetal-api-net/controllers/controllers_suite_test.go index ee4b671e..397726f5 100644 --- a/onmetal-api-net/controllers/controllers_suite_test.go +++ b/onmetal-api-net/controllers/controllers_suite_test.go @@ -24,6 +24,7 @@ import ( "time" onmetalapinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/v1alpha1" + onmetalapinet "github.com/onmetal/onmetal-api-net/onmetal-api-net/controllers/certificate/onmetal-api-net" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go4.org/netipx" @@ -37,7 +38,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" //+kubebuilder:scaffold:imports ) @@ -81,7 +81,7 @@ const ( ) var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + logf.SetLogger(GinkgoLogr) By("bootstrapping test environment") testEnv = &envtest.Environment{ @@ -94,6 +94,7 @@ var _ = BeforeSuite(func() { cfg, err = testEnv.Start() Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) + DeferCleanup(testEnv.Stop) err = onmetalapinetv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) @@ -105,20 +106,47 @@ var _ = BeforeSuite(func() { Expect(k8sClient).NotTo(BeNil()) SetClient(k8sClient) -}) -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Host: "127.0.0.1", + MetricsBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + + // register reconciler here + Expect((&PublicIPReconciler{ + EventRecorder: &record.FakeRecorder{}, + Client: k8sManager.GetClient(), + APIReader: k8sManager.GetAPIReader(), + InitialAvailableIPs: InitialAvailableIPs(), + }).SetupWithManager(k8sManager)).To(Succeed()) + + Expect((&NetworkReconciler{ + EventRecorder: &record.FakeRecorder{}, + Client: k8sManager.GetClient(), + APIReader: k8sManager.GetAPIReader(), + MinVNI: MinVNI, + MaxVNI: MaxVNI, + }).SetupWithManager(k8sManager)).To(Succeed()) + + Expect((&CertificateApprovalReconciler{ + Client: k8sManager.GetClient(), + Recognizers: onmetalapinet.Recognizers, + }).SetupWithManager(k8sManager)).To(Succeed()) + + mgrCtx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + go func() { + defer GinkgoRecover() + Expect(k8sManager.Start(mgrCtx)).To(Succeed(), "failed to start manager") + }() }) -func SetupTest(ctx context.Context) *corev1.Namespace { +func SetupTest() *corev1.Namespace { ns := &corev1.Namespace{} - BeforeEach(func() { - mgrCtx, cancel := context.WithCancel(ctx) - DeferCleanup(cancel) + BeforeEach(func(ctx SpecContext) { *ns = corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -127,39 +155,11 @@ func SetupTest(ctx context.Context) *corev1.Namespace { } Expect(k8sClient.Create(ctx, ns)).To(Succeed(), "failed to create test namespace") - DeferCleanup(func() { + DeferCleanup(func(ctx context.Context) { Expect(k8sClient.DeleteAllOf(ctx, &onmetalapinetv1alpha1.Network{}, client.InNamespace(ns.Name))).To(Succeed(), "failed to delete networks") Expect(k8sClient.DeleteAllOf(ctx, &onmetalapinetv1alpha1.PublicIP{}, client.InNamespace(ns.Name))).To(Succeed(), "failed to delete public ips") Expect(k8sClient.Delete(ctx, ns)).To(Succeed(), "failed to delete test namespace") }) - - k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, - Host: "127.0.0.1", - MetricsBindAddress: "0", - }) - Expect(err).ToNot(HaveOccurred()) - - // register reconciler here - Expect((&PublicIPReconciler{ - EventRecorder: &record.FakeRecorder{}, - Client: k8sManager.GetClient(), - APIReader: k8sManager.GetAPIReader(), - InitialAvailableIPs: InitialAvailableIPs(), - }).SetupWithManager(k8sManager)).To(Succeed()) - - Expect((&NetworkReconciler{ - EventRecorder: &record.FakeRecorder{}, - Client: k8sManager.GetClient(), - APIReader: k8sManager.GetAPIReader(), - MinVNI: MinVNI, - MaxVNI: MaxVNI, - }).SetupWithManager(k8sManager)).To(Succeed()) - - go func() { - defer GinkgoRecover() - Expect(k8sManager.Start(mgrCtx)).To(Succeed(), "failed to start manager") - }() }) return ns diff --git a/onmetal-api-net/controllers/network_controller_test.go b/onmetal-api-net/controllers/network_controller_test.go index 0ead08fc..eb1b65bd 100644 --- a/onmetal-api-net/controllers/network_controller_test.go +++ b/onmetal-api-net/controllers/network_controller_test.go @@ -16,7 +16,6 @@ package controllers import ( onmetalapinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/v1alpha1" - . "github.com/onmetal/onmetal-api/utils/testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" @@ -27,10 +26,9 @@ import ( ) var _ = Describe("NetworkController", func() { - ctx := SetupContext() - ns := SetupTest(ctx) + ns := SetupTest() - It("should allocate a vni", func() { + It("should allocate a vni", func(ctx SpecContext) { By("creating a network") network := &onmetalapinetv1alpha1.Network{ ObjectMeta: metav1.ObjectMeta{ @@ -45,7 +43,7 @@ var _ = Describe("NetworkController", func() { Eventually(Object(network)).Should(BeAllocatedNetwork()) }) - It("should mark networks as pending if they can't be allocated and allocate them as soon as there's space", func() { + It("should mark networks as pending if they can't be allocated and allocate them as soon as there's space", func(ctx SpecContext) { By("creating networks until we run out of vnis") networkKeys := make([]client.ObjectKey, NoOfVNIs) for i := 0; i < NoOfVNIs; i++ { diff --git a/onmetal-api-net/controllers/publicip_controller_test.go b/onmetal-api-net/controllers/publicip_controller_test.go index 4d49735e..89a32bf6 100644 --- a/onmetal-api-net/controllers/publicip_controller_test.go +++ b/onmetal-api-net/controllers/publicip_controller_test.go @@ -16,7 +16,6 @@ package controllers import ( onmetalapinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/v1alpha1" - . "github.com/onmetal/onmetal-api/utils/testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" @@ -27,10 +26,9 @@ import ( ) var _ = Describe("PublicIPController", func() { - ctx := SetupContext() - ns := SetupTest(ctx) + ns := SetupTest() - It("should allocate a public ip", func() { + It("should allocate a public ip", func(ctx SpecContext) { By("creating a public ip") publicIP := &onmetalapinetv1alpha1.PublicIP{ ObjectMeta: metav1.ObjectMeta{ @@ -47,7 +45,7 @@ var _ = Describe("PublicIPController", func() { Eventually(Object(publicIP)).Should(BeAllocatedPublicIP()) }) - It("should mark public ips as pending if they can't be allocated and allocate them as soon as there's space", func() { + It("should mark public ips as pending if they can't be allocated and allocate them as soon as there's space", func(ctx SpecContext) { By("creating public ips until we run out of addresses") publicIPKeys := make([]client.ObjectKey, NoOfIPv4Addresses) for i := 0; i < NoOfIPv4Addresses; i++ { diff --git a/onmetal-api-net/main.go b/onmetal-api-net/main.go index d4027892..544e6d38 100644 --- a/onmetal-api-net/main.go +++ b/onmetal-api-net/main.go @@ -25,6 +25,7 @@ import ( onmetalapinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/v1alpha1" netflag "github.com/onmetal/onmetal-api-net/flag" "github.com/onmetal/onmetal-api-net/onmetal-api-net/controllers" + onmetalapinet "github.com/onmetal/onmetal-api-net/onmetal-api-net/controllers/certificate/onmetal-api-net" flag "github.com/spf13/pflag" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -127,6 +128,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "PublicIP") os.Exit(1) } + + if err = (&controllers.CertificateApprovalReconciler{ + Client: mgr.GetClient(), + Recognizers: onmetalapinet.Recognizers, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CertificateApproval") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {