Merge branch 'improvement/sync-kube-manifest-with-kubeadm' into tmp/o…

…ctopus/w/2.9/improvement/sync-kube-manifest-with-kubeadm

CHANGELOG.md

@@ -46,7 +46,7 @@
 - Improve error handling when providing invalid CA minion in Bootstrap
   configuration file
   (PR [#3065](https://github.com/scality/metalk8s/pull/3065))
-  
+
 - [#2925](https://github.com/scality/metalk8s/issues/2925) - Compare node metrics with average from MetalK8s UI
   (PR [#3078](https://github.com/scality/metalk8s/pull/3078))
 
@@ -61,6 +61,9 @@
 - Improve the upgrade robustness when the platform is a bit slow
   (PR [#3105](https://github.com/scality/metalk8s/pull/3105))
 
+- Use HTTPS endpoints for kube-controller-manager and kube-scheduler
+  (PR [#3152](https://github.com/scality/metalk8s/pull/3152))
+
 ### Bug fixes
 
 - [#3079](https://github.com/scality/metalk8s/issues/3079) - Ensure configured

charts/kube-prometheus-stack.yaml

@@ -221,3 +221,19 @@ kubeEtcd:
   service:
     port: 2381
     targetPort: 2381
+
+kubeScheduler:
+  service:
+    port: 10259
+    targetPort: 10259
+  serviceMonitor:
+    https: true
+    insecureSkipVerify: true
+
+kubeControllerManager:
+  service:
+    port: 10257
+    targetPort: 10257
+  serviceMonitor:
+    https: true
+    insecureSkipVerify: true

salt/metalk8s/addons/prometheus-operator/deployed/chart.sls

@@ -53263,9 +53263,9 @@ spec:
   clusterIP: None
   ports:
   - name: http-metrics
-    port: 10252
+    port: 10257
     protocol: TCP
-    targetPort: 10252
+    targetPort: 10257
   selector:
     component: kube-controller-manager
   type: ClusterIP
@@ -53338,9 +53338,9 @@ spec:
   clusterIP: None
   ports:
   - name: http-metrics
-    port: 10251
+    port: 10259
     protocol: TCP
-    targetPort: 10251
+    targetPort: 10259
   selector:
     component: kube-scheduler
   type: ClusterIP
@@ -56468,6 +56468,10 @@ spec:
   endpoints:
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     port: http-metrics
+    scheme: https
+    tlsConfig:
+      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      insecureSkipVerify: true
   jobLabel: jobLabel
   namespaceSelector:
     matchNames:
@@ -56546,6 +56550,10 @@ spec:
   endpoints:
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     port: http-metrics
+    scheme: https
+    tlsConfig:
+      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      insecureSkipVerify: true
   jobLabel: jobLabel
   namespaceSelector:
     matchNames:

salt/metalk8s/kubernetes/apiserver/installed.sls

@@ -52,16 +52,17 @@ Create kube-apiserver Pod manifest:
         image_name: {{ build_image_name("kube-apiserver") }}
         port: 6443
         scheme: HTTPS
+        liveness_path: /livez
+        readiness_path: /readyz
         command:
+        # kubeadm flags {
           - kube-apiserver
-          - --authorization-mode=Node,RBAC
           - --advertise-address={{ host }}
           - --allow-privileged=true
+          - --authorization-mode=Node,RBAC
           - --client-ca-file=/etc/kubernetes/pki/ca.crt
-          - --cors-allowed-origins=.*
           - --enable-admission-plugins=NodeRestriction
           - --enable-bootstrap-token-auth=true
-          - --encryption-provider-config={{ encryption_k8s_path }}
           - --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
           - --etcd-certfile={{ certificates.client.files['apiserver-etcd'].path }}
           - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
@@ -84,6 +85,9 @@ Create kube-apiserver Pod manifest:
           - --service-cluster-ip-range={{ networks.service }}
           - --tls-cert-file={{ certificates.server.files.apiserver.path }}
           - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
+        # }
+          - --encryption-provider-config={{ encryption_k8s_path }}
+          - --cors-allowed-origins=.*
           - --oidc-issuer-url=https://{{ ingress_control_plane }}/oidc
           - --oidc-client-id=oidc-auth-client
           - --oidc-ca-file=/etc/metalk8s/pki/nginx-ingress/ca.crt

salt/metalk8s/kubernetes/controller-manager/installed.sls

@@ -20,24 +20,35 @@ Create kube-controller-manager Pod manifest:
         image_name: {{ build_image_name("kube-controller-manager") }}
         host: {{ grains['metalk8s']['control_plane_ip'] }}
         port: http-metrics
-        scheme: HTTP
+        scheme: HTTPS
         command:
+        # kubeadm flags {
           - kube-controller-manager
-          - --address={{ grains['metalk8s']['control_plane_ip'] }}
-          - --allocate-node-cidrs=true
-          - --cluster-cidr={{ networks.pod }}
+          - --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf
+          - --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf
+          - --bind-address={{ grains['metalk8s']['control_plane_ip'] }}
+          - --client-ca-file=/etc/kubernetes/pki/ca.crt
+          - --cluster-name=kubernetes
+          # In MetalK8s we do not use TLS bootstrapping
+          #- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
+          #- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
           - --controllers=*,bootstrapsigner,tokencleaner
           - --kubeconfig=/etc/kubernetes/controller-manager.conf
           - --leader-elect=true
-          - --node-cidr-mask-size=24
+          - --port=0
+          - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
           - --root-ca-file=/etc/kubernetes/pki/ca.crt
           - --service-account-private-key-file=/etc/kubernetes/pki/sa.key
           - --use-service-account-credentials=true
+        # }
+          - --allocate-node-cidrs=true
+          - --cluster-cidr={{ networks.pod }}
+          - --node-cidr-mask-size=24
           - --v={{ 2 if metalk8s.debug else 0 }}
         requested_cpu: 200m
         ports:
           - name: http-metrics
-            containerPort: 10252
+            containerPort: 10257
         volumes:
           {%- if grains['os_family'] == 'RedHat' %}
           - path: /etc/pki

salt/metalk8s/kubernetes/etcd/files/manifest.yaml

@@ -20,19 +20,17 @@ spec:
         - {{ part }}
       {%- endfor %}
       livenessProbe:
-        exec:
-          command:
-            - /bin/sh
-            - -ec
-            - ETCDCTL_API=3 etcdctl
-              --endpoints=https://[127.0.0.1]:2379
-              --cacert=/etc/kubernetes/pki/etcd/ca.crt
-              --cert={{ etcd_healthcheck_cert }}
-              --key=/etc/kubernetes/pki/etcd/healthcheck-client.key
-              get foo
         failureThreshold: 8
+        httpGet:
+          host: 127.0.0.1
+          path: /health
+          port: http-metrics
+          scheme: HTTP
         initialDelaySeconds: 15
         timeoutSeconds: 15
+      ports:
+        - name: http-metrics
+          containerPort: 2381
       resources: {}
       volumeMounts:
         {%- for volume in volumes %}

salt/metalk8s/kubernetes/etcd/installed.sls

@@ -59,25 +59,27 @@ Create local etcd Pod manifest:
         name: etcd
         image_name: {{ build_image_name('etcd') }}
         command:
+        # kubeadm flags {
           - etcd
           - --advertise-client-urls=https://{{ node_ip }}:2379
           - --cert-file={{ certificates.server.files.etcd.path }}
           - --client-cert-auth=true
           - --data-dir=/var/lib/etcd
           - --initial-advertise-peer-urls=https://{{ node_ip }}:2380
           - --initial-cluster={{ etcd_initial_cluster| sort | join(',') }}
-          - --initial-cluster-state={{ state }}
           - --key-file=/etc/kubernetes/pki/etcd/server.key
           - --listen-client-urls=https://127.0.0.1:2379,https://{{ node_ip }}:2379
-          - --listen-peer-urls=https://{{ node_ip }}:2380
           - --listen-metrics-urls=http://127.0.0.1:2381,http://{{ node_ip }}:2381
+          - --listen-peer-urls=https://{{ node_ip }}:2380
           - --name={{ node_name }}
           - --peer-cert-file={{ certificates.server.files['etcd-peer'].path }}
           - --peer-client-cert-auth=true
           - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
           - --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
           - --snapshot-count=10000
           - --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
+        # }
+          - --initial-cluster-state={{ state }}
         volumes:
           - path: /var/lib/etcd
             name: etcd-data

salt/metalk8s/kubernetes/files/control-plane-manifest.yaml

@@ -24,11 +24,24 @@ spec:
         failureThreshold: 8
         httpGet:
           host: {{ host }}
+        {%- if liveness_path is defined and liveness_path %}
+          path: {{ liveness_path }}
+        {%- else %}
           path: /healthz
+        {%- endif %}
           port: {{ port }}
           scheme: {{ scheme }}
         initialDelaySeconds: 15
         timeoutSeconds: 15
+    {%- if readiness_path is defined and readiness_path %}
+      readinessProbe:
+        failureThreshold: 3
+        httpGet:
+          host: {{ host }}
+          path: {{ readiness_path }}
+          port: {{ port }}
+          scheme: {{ scheme }}
+    {%- endif %}
 {%- if ports is defined and ports %}
       ports:
       {{ ports | yaml(False) | indent(6, False) }}

salt/metalk8s/kubernetes/scheduler/installed.sls

@@ -15,17 +15,23 @@ Create kube-scheduler Pod manifest:
         image_name: {{ build_image_name("kube-scheduler") }}
         host: {{ grains['metalk8s']['control_plane_ip'] }}
         port: http-metrics
-        scheme: HTTP
+        scheme: HTTPS
         command:
+        # kubeadm flags {
           - kube-scheduler
-          - --address={{ grains['metalk8s']['control_plane_ip'] }}
+          - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
+          - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
+          - --bind-address={{ grains['metalk8s']['control_plane_ip'] }}
           - --kubeconfig=/etc/kubernetes/scheduler.conf
           - --leader-elect=true
+          - --port=0
+        # }
+          - --bind-address={{ grains['metalk8s']['control_plane_ip'] }}
           - --v={{ 2 if metalk8s.debug else 0 }}
         requested_cpu: 100m
         ports:
           - name: http-metrics
-            containerPort: 10251
+            containerPort: 10259
         volumes:
           - path: /etc/kubernetes/scheduler.conf
             name: kubeconfig

tools/get-kubeadm-manifests/Dockerfile

@@ -0,0 +1,26 @@
+FROM centos:7
+
+ARG K8S_VERSION
+
+RUN printf "[kubernetes]\n\
+name=Kubernetes\n\
+baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64\n\
+enabled=1\n\
+gpgcheck=1\n\
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg\n\
+       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg\n" \
+    > /etc/yum.repos.d/kubernetes.repo \
+    && yum install -y "kubeadm-${K8S_VERSION}" --disableexcludes=kubernetes \
+    && yum clean all
+
+RUN kubeadm init phase control-plane all \
+    && kubeadm init phase etcd local
+
+CMD echo -e "\e[32m#### Apiserver manifest ####\e[39m" \
+    && cat /etc/kubernetes/manifests/kube-apiserver.yaml \
+    && echo -e "\e[32m#### Controller manager manifest ####\e[39m" \
+    && cat /etc/kubernetes/manifests/kube-controller-manager.yaml \
+    && echo -e "\e[32m#### Scheduler manifest ####\e[39m" \
+    && cat /etc/kubernetes/manifests/kube-scheduler.yaml \
+    && echo -e "\e[32m#### Etcd manifest ####\e[39m" \
+    && cat /etc/kubernetes/manifests/etcd.yaml

tools/get-kubeadm-manifests/README.md

@@ -0,0 +1,20 @@
+Simple tool to get the kubernetes control plane manifests deployed by kubeadm for a
+specific version
+
+## Building
+
+Build the docker container for the Kubernetes version you want
+
+```shell
+K8S_VERSION=$(grep 'K8S_VERSION.*=' "$(git rev-parse --show-toplevel)/buildchain/buildchain/versions.py" | cut -d '"' -f 2)
+docker build "$(git rev-parse --show-toplevel)/tools/get-kubeadm-manifests" --build-arg "K8S_VERSION=$K8S_VERSION" -t "metalk8s-kubeadm:$K8S_VERSION"
+```
+
+## Get Kubernetes control plane manifests
+
+To retrieve the various Kubernetes control plane manifests,
+you must run the previously built container.
+
+```shell
+docker run -it --rm "metalk8s-kubeadm:$K8S_VERSION"
+```