Merge pull request #485 from deckhouse/containerd-nvidia

[bashible] fixed notManaged mode for CRI.

candi/bashible/common-steps/node-group/064_configure_kubelet.sh.tpl

@@ -24,6 +24,54 @@ fi
 # This folder doesn't have time to create before we stop freshly, unconfigured kubelet during bootstrap (step 034_install_kubelet_and_his_friends.sh).
 mkdir -p /var/lib/kubelet
 
+# Check CRI type and set appropriated parameters.
+# cgroup default is `systemd`, only for docker cri we use `cgroupfs`.
+cgroup_driver="systemd"
+
+{{- if eq .cri "NotManaged" }}
+  {{- if .nodeGroup.cri.notManaged.criSocketPath }}
+cri_socket_path={{ .nodeGroup.cri.notManaged.criSocketPath | quote }}
+  {{- else }}
+for socket_path in /var/run/docker.sock /run/containerd/containerd.sock; do
+  if [[ -S "${socket_path}" ]]; then
+    cri_socket_path="${socket_path}"
+    break
+  fi
+done
+  {{- end }}
+
+if [[ -z "${cri_socket_path}" ]]; then
+  bb-log-error 'CRI socket is not found, need to manually set "nodeGroup.cri.notManaged.criSocketPath"'
+  exit 1
+fi
+
+if grep -q "docker" <<< "${cri_socket_path}"; then
+  cri_type="NotManagedDocker"
+else
+  cri_type="NotManagedContainerd"
+fi
+{{- else if eq .cri "Docker" }}
+cri_type="Docker"
+{{- else }}
+cri_type="Containerd"
+{{- end }}
+
+if [[ "${cri_type}" == "Docker" || "${cri_type}" == "NotManagedDocker" ]]; then
+  cgroup_driver="cgroupfs"
+  criDir=$(docker info --format '{{`{{.DockerRootDir}}`}}')
+  if [ -d "${criDir}/overlay2" ]; then
+    criDir="${criDir}/overlay2"
+  else
+    if [ -d "${criDir}/aufs" ]; then
+      criDir="${criDir}/aufs"
+    fi
+  fi
+fi
+
+if [[ "${cri_type}" == "Containerd" || "${cri_type}" == "NotManagedContainerd" ]]; then
+  criDir=$(crictl info -o json | jq -r '.config.containerdRootDir')
+fi
+
 # Calculate eviction thresholds.
 
 # We don't need more free space on partition.
@@ -65,20 +113,6 @@ if [ "$(($nodefsInodesKFivePercent*2))" -gt "$(($needInodesFree*2))" ]; then
   evictionSoftThresholdNodefsInodesFree="$(($needInodesFree*2))k"
 fi
 
-{{- if or (eq .cri "Docker") (eq .cri "Containerd")}}
-{{- if eq .cri "Docker" }}
-criDir=$(docker info --format '{{`{{.DockerRootDir}}`}}')
-if [ -d "${criDir}/overlay2" ]; then
-  criDir="${criDir}/overlay2"
-else
-  if [ -d "${criDir}/aufs" ]; then
-    criDir="${criDir}/aufs"
-  fi
-fi
-{{- else }}
-criDir=$(/usr/local/bin/crictl info -o json | jq -r '.config.containerdRootDir')
-{{- end }}
-
 imagefsSize=$(df --output=size $criDir | tail -n1)
 imagefsSizeGFivePercent=$((imagefsSize/(1000*1000)*5/100))
 if [ "$imagefsSizeGFivePercent" -gt "$maxAvailableReservedSpace" ]; then
@@ -87,7 +121,6 @@ fi
 if [ "$(($imagefsSizeGFivePercent*2))" -gt "$(($maxAvailableReservedSpace*2))" ]; then
   evictionSoftThresholdImagefsAvailable="$(($maxAvailableReservedSpace*2))G"
 fi
-{{- end }}
 
 imagefsInodes=$(df --output=itotal $criDir | tail -n1)
 imagefsInodesKFivePercent=$((imagefsInodes/1000*5/100))
@@ -98,7 +131,6 @@ if [ "$(($imagefsInodesKFivePercent*2))" -gt "$(($needInodesFree*2))" ]; then
   evictionSoftThresholdImagefsInodesFree="$(($needInodesFree*2))k"
 fi
 
-
 bb-sync-file /var/lib/kubelet/config.yaml - << EOF
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
@@ -117,11 +149,7 @@ authorization:
     cacheUnauthorizedTTL: 30s
 cgroupRoot: "/"
 cgroupsPerQOS: true
-{{- if eq .cri "Containerd" }}
-cgroupDriver: systemd
-{{- else }}
-cgroupDriver: cgroupfs
-{{- end }}
+cgroupDriver: ${cgroup_driver}
 {{- if eq .runType "Normal" }}
 clusterDomain: {{ .normal.clusterDomain }}
 clusterDNS:

candi/bashible/common-steps/node-group/068_configure_kubelet_systemd_unit.sh.tpl

@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Миграция!!!!! Удалить после выката
-rm -f /etc/systemd/system/kubelet.service.d/cim.conf
-rm -rf /var/lib/kubelet/manifests
-
 # In case we adopting node bootstrapped by kubeadm
 rm -f /etc/systemd/system/kubelet.service.d/10-kubeadm.conf     # for ubuntu
 rm -f /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf # for centos
@@ -24,6 +20,34 @@ rm -f /var/lib/kubelet/kubeadm-flags.env
 # Read previously discovered IP
 discovered_node_ip="$(</var/lib/bashible/discovered-node-ip)"
 
+cri_config=""
+
+{{- if eq .cri "Containerd" }}
+cri_config="--container-runtime=remote --container-runtime-endpoint=unix:/var/run/containerd/containerd.sock"
+{{- else if eq .cri "NotManaged" }}
+  {{- if .nodeGroup.cri.notManaged.criSocketPath }}
+cri_socket_path={{ .nodeGroup.cri.notManaged.criSocketPath | quote }}
+  {{- else }}
+for socket_path in /var/run/docker.sock /run/containerd/containerd.sock; do
+  if [[ -S "${socket_path}" ]]; then
+    cri_socket_path="${socket_path}"
+    break
+  fi
+done
+  {{- end }}
+
+if [[ -z "${cri_socket_path}" ]]; then
+  bb-log-error 'CRI socket is not found, need to manually set "nodeGroup.cri.notManaged.criSocketPath"'
+  exit 1
+fi
+
+if grep -q "docker" <<< "${cri_socket_path}"; then
+  cri_config="--container-runtime=docker --docker-endpoint=unix://${cri_socket_path}"
+else
+  cri_config="--container-runtime=remote --container-runtime-endpoint=unix:${cri_socket_path}"
+fi
+{{- end }}
+
 bb-event-on 'bb-sync-file-changed' '_enable_kubelet_service'
 function _enable_kubelet_service() {
 {{- if ne .runType "ImageBuilding" }}
@@ -65,9 +89,6 @@ $([ -n "$discovered_node_ip" ] && echo -e "\n    --node-ip=${discovered_node_ip}
 {{- if hasKey .nodeGroup "kubelet" }}
     --root-dir={{ .nodeGroup.kubelet.rootDir | default "/var/lib/kubelet" }} \\
 {{- end }}
-{{- if eq .cri "Containerd" }}
-    --container-runtime=remote \\
-    --container-runtime-endpoint=unix:/var/run/containerd/containerd.sock \\
-{{- end }}
+    ${cri_config} \\
     --v=2
 EOF

candi/openapi/doc-ru-node_group.yaml

@@ -306,6 +306,13 @@ spec:
                         manage:
                           description: |
                             Автоматическое управление версией и параметрами Docker.
+                    notManaged: &notManaged
+                      type: object
+                      description: Настройки для cri установленных на нодах вручную.
+                      properties:
+                        criSocketPath:
+                          type: string
+                          description: Путь к сокету cri.
                 cloudInstances: *cloudInstances
                 nodeTemplate: *nodeTemplate
                 chaos: *chaos
@@ -355,6 +362,7 @@ spec:
                           description: Максимальное количество потоков одновременного скачивания Docker-образов.
                         manage:
                           description: Автоматическое управление версией и параметрами Docker.
+                    notManaged: *notManaged
                 cloudInstances: *cloudInstances
                 nodeTemplate: *nodeTemplate
                 chaos: *chaos

candi/openapi/node_group.yaml

@@ -546,6 +546,13 @@ spec:
                           type: boolean
                           x-doc-default: true
                           description: Enable Docker maintenance from bashible.
+                    notManaged: &notManaged
+                      type: object
+                      description: Settings for not managed cri for nodes.
+                      properties:
+                        criSocketPath:
+                          type: string
+                          description: Path to cri socket.
                   oneOf:
                     - properties:
                         type:
@@ -621,6 +628,7 @@ spec:
                           type: boolean
                           x-doc-default: true
                           description: Enable Docker maintenance from bashible.
+                    notManaged: *notManaged
                   oneOf:
                     - properties:
                         type:

modules/040-node-manager/hooks/internal/v1/nodegroup.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2021 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package v1
 
 import (
@@ -82,6 +98,9 @@ type CRI struct {
 
 	// Docker settings for nodes.
 	Docker *Docker `json:"docker,omitempty"`
+
+	// NotManaged settings for nodes.
+	NotManaged *NotManaged `json:"notManaged,omitempty"`
 }
 
 func (c CRI) IsEmpty() bool {
@@ -101,6 +120,11 @@ type Docker struct {
 	Manage *bool `json:"manage,omitempty"`
 }
 
+type NotManaged struct {
+	// Set custom path to CRI socket
+	CriSocketPath *string `json:"criSocketPath,omitempty"`
+}
+
 // CloudInstances is an extra parameters for NodeGroup with type Cloud.
 type CloudInstances struct {
 	// List of availability zones to create instances in.

modules/040-node-manager/hooks/internal/v1alpha2/nodegroup.go

@@ -81,6 +81,9 @@ type CRI struct {
 
 	// Docker settings for nodes.
 	Docker *Docker `json:"docker,omitempty"`
+
+	// NotManaged settings for nodes.
+	NotManaged *NotManaged `json:"notManaged,omitempty"`
 }
 
 func (c CRI) IsEmpty() bool {
@@ -100,6 +103,11 @@ type Docker struct {
 	Manage *bool `json:"manage,omitempty"`
 }
 
+type NotManaged struct {
+	// Set custom path to CRI socket
+	CriSocketPath *string `json:"criSocketPath,omitempty"`
+}
+
 // CloudInstances is an extra parameters for NodeGroup with type Cloud.
 type CloudInstances struct {
 	// List of availability zones to create instances in.