Skip to content

Commit

Permalink
Add CA secret endpoint for kubeadm (#873)
Browse files Browse the repository at this point in the history
* Add CA secret endpoint for kubeadm

* Pass seed kubeadm helm value

* Add kubeadm seed template

* add kubeadm values

* Add helm values

* base64 encode cert

* Fix encoding

* fix cluster name

* add buildx target

* do not apply csi for kubeadm clusters

* add kubeadm image repo override

---------

Co-authored-by: Dmitri Fedotov <dmitri.fedotov@sap.com>
  • Loading branch information
jknipper and defo89 committed Feb 19, 2024
1 parent 29baad2 commit 0c0d531
Show file tree
Hide file tree
Showing 29 changed files with 1,958 additions and 9 deletions.
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ build:
docker build $(BUILD_ARGS) -t sapcc/kubernikus-docs:$(VERSION) -f Dockerfile.kubernikus-docs .
docker build $(BUILD_ARGS) -t sapcc/kubernikus:$(VERSION) -f Dockerfile .

buildx:
docker buildx build $(BUILD_ARGS) --provenance=false --platform=linux/amd64 -t sapcc/kubernikus-binaries:$(VERSION) -f Dockerfile.kubernikus-binaries .
docker buildx build $(BUILD_ARGS) --provenance=false --platform=linux/amd64 -t sapcc/kubernikus-docs-builder:$(VERSION) --cache-from=sapcc/kubernikus-docs-builder:latest ./contrib/kubernikus-docs-builder
docker buildx build $(BUILD_ARGS) --provenance=false --platform=linux/amd64 -t sapcc/kubernikusctl:$(VERSION) ./contrib/kubernikusctl
docker buildx build $(BUILD_ARGS) --provenance=false --platform=linux/amd64 -t sapcc/kubernikus-docs:$(VERSION) -f Dockerfile.kubernikus-docs .
docker buildx build $(BUILD_ARGS) --push --provenance=false --platform=linux/amd64 -t sapcc/kubernikus:$(VERSION) -f Dockerfile .

pull:
docker pull sapcc/kubernikus-docs-builder:latest

Expand Down
2 changes: 1 addition & 1 deletion charts/seed/templates/csi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ region="{{ .Values.openstack.region }}"
rescan-on-resize = yes
` -}}
{{- if semverCompare ">= 1.20-0" .Capabilities.KubeVersion.Version -}}
{{- if and (not .Values.seedKubeadm) (semverCompare ">= 1.20-0" .Capabilities.KubeVersion.Version) -}}
apiVersion: v1
kind: Secret
metadata:
Expand Down
265 changes: 265 additions & 0 deletions charts/seed/templates/kubeadm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,265 @@
{{- if .Values.seedKubeadm }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: cc-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: User
name: {{ .Values.clusterAdminUser }}
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubeadm:bootstrap-signer-clusterinfo
namespace: kube-public
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubeadm:bootstrap-signer-clusterinfo
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:anonymous
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kubeadm:bootstrap-signer-clusterinfo
namespace: kube-public
rules:
- apiGroups:
- ""
resourceNames:
- cluster-info
resources:
- configmaps
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kubeadm:nodes-kubeadm-config
namespace: kube-system
rules:
- apiGroups:
- ""
resourceNames:
- kubeadm-config
resources:
- configmaps
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubeadm:nodes-kubeadm-config
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubeadm:nodes-kubeadm-config
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:kubeadm:default-node-token
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kube-proxy
namespace: kube-system
rules:
- apiGroups:
- ""
resourceNames:
- kube-proxy
resources:
- configmaps
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kubeadm:kubelet-config
namespace: kube-system
rules:
- apiGroups:
- ""
resourceNames:
- kubelet-config
resources:
- configmaps
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kube-proxy
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kube-proxy
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:kubeadm:default-node-token
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubeadm:kubelet-config
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubeadm:kubelet-config
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:kubeadm:default-node-token
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kubeadm:get-nodes
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubeadm:get-nodes
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kubeadm:get-nodes
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:kubeadm:default-node-token
---
apiVersion: v1
kind: ConfigMap
metadata:
name: cluster-info
namespace: kube-public
data:
kubeconfig : |
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: {{ .Values.tlsCaCert | b64enc }}
server: https://{{ .Values.api.apiserverHost }}
name: ""
contexts: null
current-context: ""
kind: Config
preferences: {}
users: null
---
apiVersion: v1
data:
ClusterConfiguration: |
apiServer:
extraArgs:
authorization-mode: Node,RBAC
cloud-provider: external
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: "{{ .Values.name }}"
controlPlaneEndpoint: "{{ .Values.api.apiserverHost }}"
controllerManager:
extraArgs:
cloud-provider: external
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: {{ .Values.kubeadmImageRepository }}
kind: ClusterConfiguration
kubernetesVersion: {{ .Capabilities.KubeVersion.Version }}
networking:
dnsDomain: {{ .Values.dns.domain }}
podSubnet: {{ .Values.clusterCIDR }}
serviceSubnet: {{ .Values.serviceCIDR }}
scheduler: {}
kind: ConfigMap
metadata:
name: kubeadm-config
namespace: kube-system
---
apiVersion: v1
data:
kubelet: |
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 0s
cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
- {{ .Values.dns.address }}
clusterDomain: {{ .Values.dns.domain }}
containerRuntimeEndpoint: ""
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging:
flushFrequency: 0
options:
json:
infoBufferSize: "0"
verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
resolvConf: /run/systemd/resolve/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
kind: ConfigMap
metadata:
name: kubelet-config
namespace: kube-system
{{- end }}
31 changes: 31 additions & 0 deletions charts/seed/templates/proxy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,37 @@ data:
mode: "iptables"
oomScoreAdj: -999
portRange: ""
{{- if .Values.seedKubeadm }}
config.conf: |
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
healthzBindAddress: 0.0.0.0:10256
metricsBindAddress: 0.0.0.0:10249
clientConnection:
acceptContentTypes: ""
contentType: application/vnd.kubernetes.protobuf
kubeconfig: /var/lib/kube-proxy/kubeconfig
qps: 5
burst: 10
clusterCIDR: {{ .Values.clusterCIDR }}
configSyncPeriod: 15m0s
conntrack:
maxPerCore: 32768
min: 131072
tcpCloseWaitTimeout: 1h0m0s
tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
featureGates: {}
iptables:
masqueradeAll: false
masqueradeBit: 14
minSyncPeriod: 0s
syncPeriod: 30s
mode: "iptables"
oomScoreAdj: -999
portRange: ""
{{- end }}
---
apiVersion: apps/v1
kind: DaemonSet
Expand Down
5 changes: 5 additions & 0 deletions charts/seed/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ version: {}
# kubernikus:
# kubernetes: 1.10.11

clusterAdminUser: TCC_D038720_01
kubeadmImageRepository: keppel.global.cloud.sap/ccloud-registry-k8s-io-mirror
# tlsCaCert:
# serviceCIDR

api:
replicaCount: 1
# apiserverHost:
Expand Down
3 changes: 2 additions & 1 deletion etc/policy-ccadmin.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,6 @@
"GetClusterEvents": "rule:kubernetes_user",
"GetClusterInfo": "rule:kubernetes_user",
"GetBootstrapConfig": "rule:kubernetes_admin",
"GetClusterValues": "rule:kubernetes_cloud_admin"
"GetClusterValues": "rule:kubernetes_cloud_admin",
"GetClusterKubeadmSecret": "rule:kubernetes_admin"
}
3 changes: 2 additions & 1 deletion etc/policy.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,6 @@
"GetClusterEvents": "rule:kubernetes_user",
"GetClusterInfo": "rule:kubernetes_user",
"GetBootstrapConfig": "rule:kubernetes_admin",
"GetClusterValues": "rule:kubernetes_cloud_admin"
"GetClusterValues": "rule:kubernetes_cloud_admin",
"GetClusterKubeadmSecret": "rule:kubernetes_admin"
}

0 comments on commit 0c0d531

Please sign in to comment.