From 651dd7526f70215a138a690d385b66b48bd9b50e Mon Sep 17 00:00:00 2001 From: Maciej Wereski Date: Wed, 24 Feb 2021 15:30:15 +0100 Subject: [PATCH 1/2] Add KubeSchedulerConfiguration for k8s 1.19 and up With release of version 1.19.0 of kubernetes KubeSchedulerConfiguration was graduated to beta. It allows to extend different stages of scheduling with profiles. Such effect is achieved by using plugins and extensions. This patch adds KubeSchedulerConfiguration for versions 1.19 and later. Configuration is set to k8s defaults or to kubespray vars. Moving those defaults to new vars will be done in following patch. Signed-off-by: Maciej Wereski --- roles/kubernetes/control-plane/tasks/main.yml | 7 +++++++ .../templates/kubeadm-config.v1beta2.yaml.j2 | 12 +++++++++++- .../templates/kubescheduler-config.v1beta1.yaml.j2 | 9 +++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 roles/kubernetes/control-plane/templates/kubescheduler-config.v1beta1.yaml.j2 diff --git a/roles/kubernetes/control-plane/tasks/main.yml b/roles/kubernetes/control-plane/tasks/main.yml index 6fba951c28d..a32644f185e 100644 --- a/roles/kubernetes/control-plane/tasks/main.yml +++ b/roles/kubernetes/control-plane/tasks/main.yml @@ -15,6 +15,13 @@ dest: "{{ kube_config_dir }}/webhook-authorization-config.yaml" when: kube_webhook_authorization|default(false) +- name: Create kube-scheduler config + template: + src: kubescheduler-config.v1beta1.yaml.j2 + dest: "{{ kube_config_dir }}/kubescheduler-config.yaml" + mode: 0644 + when: kube_version is version('v1.19.0', '>=') + - import_tasks: encrypt-at-rest.yml when: - kube_encrypt_secret_data diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 index b60c1dce62e..7c0761647ef 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 @@ -327,8 +327,12 @@ controllerManager: scheduler: extraArgs: bind-address: {{ kube_scheduler_bind_address }} +{% if kube_version is version('v1.19.0', '>=') %} + config: {{ kube_config_dir }}/kubescheduler-config.yaml +{% else %} leader-elect-lease-duration: {{ kube_scheduler_leader_elect_lease_duration }} leader-elect-renew-deadline: {{ kube_scheduler_leader_elect_renew_deadline }} +{% endif %} {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} @@ -344,8 +348,14 @@ scheduler: tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %} {% endif %} -{% if scheduler_extra_volumes %} +{% if scheduler_extra_volumes or kube_version is version('v1.19.0', '>=') %} extraVolumes: +{% if kube_version is version('v.1.19.0', '>=') %} + - name: kubescheduler-config + hostPath: {{ kube_config_dir }}/kubescheduler-config.yaml + mountPath: {{ kube_config_dir }}/kubescheduler-config.yaml + readOnly: true +{% endif %} {% for volume in scheduler_extra_volumes %} - name: {{ volume.name }} hostPath: {{ volume.hostPath }} diff --git a/roles/kubernetes/control-plane/templates/kubescheduler-config.v1beta1.yaml.j2 b/roles/kubernetes/control-plane/templates/kubescheduler-config.v1beta1.yaml.j2 new file mode 100644 index 00000000000..5fde9a13f8f --- /dev/null +++ b/roles/kubernetes/control-plane/templates/kubescheduler-config.v1beta1.yaml.j2 @@ -0,0 +1,9 @@ +apiVersion: kubescheduler.config.k8s.io/v1beta1 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "{{ kube_config_dir }}/scheduler.conf" +extenders: +leaderElection: + leaseDuration: {{ kube_scheduler_leader_elect_lease_duration }} + renewDeadline: {{ kube_scheduler_leader_elect_renew_deadline }} +profiles: From 7faa577eec24406adbdc29d75b075fb85a8084d7 Mon Sep 17 00:00:00 2001 From: Maciej Wereski Date: Wed, 3 Mar 2021 15:50:20 +0100 Subject: [PATCH 2/2] KubeSchedulerConfiguration: add defaults Signed-off-by: Maciej Wereski --- .../defaults/main/kube-scheduler.yml | 33 +++++++++++++++++++ .../control-plane/defaults/main/main.yml | 8 +---- .../kubescheduler-config.v1beta1.yaml.j2 | 15 +++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml diff --git a/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml b/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml new file mode 100644 index 00000000000..e61bcb7725c --- /dev/null +++ b/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml @@ -0,0 +1,33 @@ +--- +# Extra args passed by kubeadm +kube_kubeadm_scheduler_extra_args: {} + +# Associated interface must be reachable by the rest of the cluster, and by +# CLI/web clients. +kube_scheduler_bind_address: 0.0.0.0 + +# ClientConnection options (e.g. Burst, QPS) except from kubeconfig. +kube_scheduler_client_conn_extra_opts: {} + +# Additional KubeSchedulerConfiguration settings (e.g. metricsBindAddress). +kube_scheduler_config_extra_opts: {} + +# List of scheduler extenders (dicts), each holding the values of how to +# communicate with the extender. +kube_scheduler_extenders: [] + +# Leader Election options (e.g. ResourceName, RetryPerion) except from +# LeaseDuration and Renew deadline which are defined in following vars. +kube_scheduler_leader_elect_extra_opts: {} + +# Leader election lease duration +kube_scheduler_leader_elect_lease_duration: 15s + +# Leader election lease timeout +kube_scheduler_leader_elect_renew_deadline: 10s + +# Lisf of scheduling profiles (ditcs) supported by kube-scheduler +kube_scheduler_profiles: [] + +# Extra volume mounts +scheduler_extra_volumes: {} diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index c671326dd7c..291678e1e20 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -26,15 +26,11 @@ kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem # Associated interfaces must be reachable by the rest of the cluster, and by # CLI/web clients. kube_controller_manager_bind_address: 0.0.0.0 -kube_scheduler_bind_address: 0.0.0.0 -# Leader election lease durations and timeouts for scheduler and controller-manager +# Leader election lease durations and timeouts for controller-manager kube_controller_manager_leader_elect_lease_duration: 15s kube_controller_manager_leader_elect_renew_deadline: 10s -kube_scheduler_leader_elect_lease_duration: 15s -kube_scheduler_leader_elect_renew_deadline: 10s - # discovery_timeout modifies the discovery timeout discovery_timeout: 5m0s @@ -134,7 +130,6 @@ kubelet_preferred_address_types: 'InternalDNS,InternalIP,Hostname,ExternalDNS,Ex ## Extra args for k8s components passing by kubeadm kube_kubeadm_apiserver_extra_args: {} kube_kubeadm_controller_extra_args: {} -kube_kubeadm_scheduler_extra_args: {} ## Extra control plane host volume mounts ## Example: @@ -145,7 +140,6 @@ kube_kubeadm_scheduler_extra_args: {} # readOnly: true apiserver_extra_volumes: {} controller_manager_extra_volumes: {} -scheduler_extra_volumes: {} ## Encrypting Secret Data at Rest kube_encrypt_secret_data: false diff --git a/roles/kubernetes/control-plane/templates/kubescheduler-config.v1beta1.yaml.j2 b/roles/kubernetes/control-plane/templates/kubescheduler-config.v1beta1.yaml.j2 index 5fde9a13f8f..87782fc1681 100644 --- a/roles/kubernetes/control-plane/templates/kubescheduler-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubescheduler-config.v1beta1.yaml.j2 @@ -2,8 +2,23 @@ apiVersion: kubescheduler.config.k8s.io/v1beta1 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: "{{ kube_config_dir }}/scheduler.conf" +{% for key in kube_scheduler_client_conn_extra_opts %} + {{ key }}: {{ kube_scheduler_client_conn_extra_opts[key] }} +{% endfor %} +{% if kube_scheduler_extenders %} extenders: +{{ kube_scheduler_extenders | to_nice_yaml(indent=2, width=256) }} +{% endif %} leaderElection: leaseDuration: {{ kube_scheduler_leader_elect_lease_duration }} renewDeadline: {{ kube_scheduler_leader_elect_renew_deadline }} +{% for key in kube_scheduler_leader_elect_extra_opts %} + {{ key }}: {{ kube_scheduler_leader_elect_extra_opts[key] }} +{% endfor %} +{% if kube_scheduler_profiles %} profiles: +{{ kube_scheduler_profiles | to_nice_yaml(indent=2, width=256) }} +{% endif %} +{% for key in kube_scheduler_config_extra_opts %} +{{ key }}: {{ kube_scheduler_config_extra_opts[key] }} +{% endfor %}