Add CA secret endpoint for kubeadm

Makefile

@@ -55,6 +55,13 @@ build:
 	docker build $(BUILD_ARGS) -t sapcc/kubernikus-docs:$(VERSION)         -f Dockerfile.kubernikus-docs .
 	docker build $(BUILD_ARGS) -t sapcc/kubernikus:$(VERSION)              -f Dockerfile .
 
+buildx:
+	docker buildx build $(BUILD_ARGS) --provenance=false --platform=linux/amd64 -t sapcc/kubernikus-binaries:$(VERSION)             -f Dockerfile.kubernikus-binaries .
+	docker buildx build $(BUILD_ARGS) --provenance=false --platform=linux/amd64 -t sapcc/kubernikus-docs-builder:$(VERSION) --cache-from=sapcc/kubernikus-docs-builder:latest ./contrib/kubernikus-docs-builder
+	docker buildx build $(BUILD_ARGS) --provenance=false --platform=linux/amd64 -t sapcc/kubernikusctl:$(VERSION)                                                             ./contrib/kubernikusctl
+	docker buildx build $(BUILD_ARGS) --provenance=false --platform=linux/amd64 -t sapcc/kubernikus-docs:$(VERSION)         -f Dockerfile.kubernikus-docs .
+	docker buildx build $(BUILD_ARGS) --push --provenance=false --platform=linux/amd64  -t sapcc/kubernikus:$(VERSION)              -f Dockerfile .
+
 pull:
 	docker pull sapcc/kubernikus-docs-builder:latest
 

charts/seed/templates/csi.yaml

@@ -10,7 +10,7 @@ region="{{ .Values.openstack.region }}"
 rescan-on-resize = yes
 ` -}}
 
-{{- if semverCompare ">= 1.20-0" .Capabilities.KubeVersion.Version -}}
+{{- if and (not .Values.seedKubeadm) (semverCompare ">= 1.20-0" .Capabilities.KubeVersion.Version) -}}
 apiVersion: v1
 kind: Secret
 metadata:

charts/seed/templates/kubeadm.yaml

@@ -0,0 +1,265 @@
+{{- if .Values.seedKubeadm }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cc-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: User
+  name: {{ .Values.clusterAdminUser }}
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeadm:bootstrap-signer-clusterinfo
+  namespace: kube-public
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubeadm:bootstrap-signer-clusterinfo
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:anonymous
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubeadm:bootstrap-signer-clusterinfo
+  namespace: kube-public
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - cluster-info
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubeadm:nodes-kubeadm-config
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubeadm-config
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeadm:nodes-kubeadm-config
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubeadm:nodes-kubeadm-config
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers:kubeadm:default-node-token
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - kube-proxy
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubeadm:kubelet-config
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubelet-config
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kube-proxy
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers:kubeadm:default-node-token
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeadm:kubelet-config
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubeadm:kubelet-config
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers:kubeadm:default-node-token
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeadm:get-nodes
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeadm:get-nodes
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubeadm:get-nodes
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers:kubeadm:default-node-token
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-info
+  namespace: kube-public
+data: 
+  kubeconfig : |
+    apiVersion: v1
+    clusters:
+    - cluster:
+        certificate-authority-data: {{ .Values.tlsCaCert | b64enc }}
+        server: https://{{ .Values.api.apiserverHost }}
+      name: ""
+    contexts: null
+    current-context: ""
+    kind: Config
+    preferences: {}
+    users: null
+---
+apiVersion: v1
+data:
+  ClusterConfiguration: |
+    apiServer:
+      extraArgs:
+        authorization-mode: Node,RBAC
+        cloud-provider: external
+      timeoutForControlPlane: 4m0s
+    apiVersion: kubeadm.k8s.io/v1beta3
+    certificatesDir: /etc/kubernetes/pki
+    clusterName: "{{ .Values.name }}"
+    controlPlaneEndpoint: "{{ .Values.api.apiserverHost }}"
+    controllerManager:
+      extraArgs:
+        cloud-provider: external
+    dns: {}
+    etcd:
+      local:
+        dataDir: /var/lib/etcd
+    imageRepository: {{ .Values.kubeadmImageRepository }}
+    kind: ClusterConfiguration
+    kubernetesVersion: {{ .Capabilities.KubeVersion.Version }}
+    networking:
+      dnsDomain: {{ .Values.dns.domain }}
+      podSubnet: {{ .Values.clusterCIDR }}
+      serviceSubnet: {{ .Values.serviceCIDR }}
+    scheduler: {}
+kind: ConfigMap
+metadata:
+  name: kubeadm-config
+  namespace: kube-system
+---
+apiVersion: v1
+data:
+  kubelet: |
+    apiVersion: kubelet.config.k8s.io/v1beta1
+    authentication:
+      anonymous:
+        enabled: false
+      webhook:
+        cacheTTL: 0s
+        enabled: true
+      x509:
+        clientCAFile: /etc/kubernetes/pki/ca.crt
+    authorization:
+      mode: Webhook
+      webhook:
+        cacheAuthorizedTTL: 0s
+        cacheUnauthorizedTTL: 0s
+    cgroupDriver: systemd
+    clusterDNS:
+    - {{ .Values.dns.address }}
+    clusterDomain: {{ .Values.dns.domain }}
+    containerRuntimeEndpoint: ""
+    cpuManagerReconcilePeriod: 0s
+    evictionPressureTransitionPeriod: 0s
+    fileCheckFrequency: 0s
+    healthzBindAddress: 127.0.0.1
+    healthzPort: 10248
+    httpCheckFrequency: 0s
+    imageMinimumGCAge: 0s
+    kind: KubeletConfiguration
+    logging:
+      flushFrequency: 0
+      options:
+        json:
+          infoBufferSize: "0"
+      verbosity: 0
+    memorySwap: {}
+    nodeStatusReportFrequency: 0s
+    nodeStatusUpdateFrequency: 0s
+    resolvConf: /run/systemd/resolve/resolv.conf
+    rotateCertificates: true
+    runtimeRequestTimeout: 0s
+    shutdownGracePeriod: 0s
+    shutdownGracePeriodCriticalPods: 0s
+    staticPodPath: /etc/kubernetes/manifests
+    streamingConnectionIdleTimeout: 0s
+    syncFrequency: 0s
+    volumeStatsAggPeriod: 0s
+kind: ConfigMap
+metadata:
+  name: kubelet-config
+  namespace: kube-system
+{{- end }}

charts/seed/templates/proxy.yaml

@@ -75,6 +75,37 @@ data:
     mode: "iptables"
     oomScoreAdj: -999
     portRange: ""
+{{- if .Values.seedKubeadm }}
+  config.conf: |
+    apiVersion: kubeproxy.config.k8s.io/v1alpha1
+    kind: KubeProxyConfiguration
+    bindAddress: 0.0.0.0
+    healthzBindAddress: 0.0.0.0:10256
+    metricsBindAddress: 0.0.0.0:10249
+    clientConnection:
+      acceptContentTypes: ""
+      contentType: application/vnd.kubernetes.protobuf
+      kubeconfig: /var/lib/kube-proxy/kubeconfig
+      qps: 5
+      burst: 10
+    clusterCIDR: {{ .Values.clusterCIDR }}
+    configSyncPeriod: 15m0s
+    conntrack:
+      maxPerCore: 32768
+      min: 131072
+      tcpCloseWaitTimeout: 1h0m0s
+      tcpEstablishedTimeout: 24h0m0s
+    enableProfiling: false
+    featureGates: {}
+    iptables:
+      masqueradeAll: false
+      masqueradeBit: 14
+      minSyncPeriod: 0s
+      syncPeriod: 30s
+    mode: "iptables"
+    oomScoreAdj: -999
+    portRange: ""
+{{- end }}
 ---
 apiVersion: apps/v1
 kind: DaemonSet

charts/seed/values.yaml

@@ -39,6 +39,11 @@ version: {}
 # kubernikus:
 # kubernetes: 1.10.11
 
+clusterAdminUser: TCC_D038720_01
+kubeadmImageRepository: keppel.global.cloud.sap/ccloud-registry-k8s-io-mirror
+# tlsCaCert:
+# serviceCIDR
+
 api:
   replicaCount: 1
   # apiserverHost:

etc/policy-ccadmin.json

@@ -14,5 +14,6 @@
   "GetClusterEvents": "rule:kubernetes_user",
   "GetClusterInfo": "rule:kubernetes_user",
   "GetBootstrapConfig": "rule:kubernetes_admin",
-  "GetClusterValues": "rule:kubernetes_cloud_admin"
+  "GetClusterValues": "rule:kubernetes_cloud_admin",
+  "GetClusterKubeadmSecret": "rule:kubernetes_admin"
 }

etc/policy.json

@@ -14,5 +14,6 @@
   "GetClusterEvents": "rule:kubernetes_user",
   "GetClusterInfo": "rule:kubernetes_user",
   "GetBootstrapConfig": "rule:kubernetes_admin",
-  "GetClusterValues": "rule:kubernetes_cloud_admin"
+  "GetClusterValues": "rule:kubernetes_cloud_admin",
+  "GetClusterKubeadmSecret": "rule:kubernetes_admin"
 }