[deckhouse] check kernel versions for enabled modules (#2709)

Signed-off-by: Denis Romanenko <denis.romanenko@flant.com>
Signed-off-by: Denys Romanenko <65756796+RomanenkoDenys@users.noreply.github.com>
Co-authored-by: flsixtyfour <pavel.tishkov@flant.com>
Co-authored-by: Andrey Polovov <andrey.polovov@flant.com>
Co-authored-by: Andrey Klimentyev <andrey.klimentyev@flant.com>
Co-authored-by: Maksim Nabokikh <maksim.nabokikh@flant.com>

ee/modules/110-istio/template_tests/module_test.go

@@ -22,7 +22,7 @@ func Test(t *testing.T) {
 
 const globalValues = `
 highAvailability: true
-enabledModules: ["operator-prometheus-crd","cert-manager","vertical-pod-autoscaler-crd"]
+enabledModules: ["operator-prometheus-crd","cert-manager","vertical-pod-autoscaler-crd","cni-cilium"]
 modules:
   publicDomainTemplate: "%s.example.com"
   placement:

ee/modules/110-istio/templates/control-plane/iop.yaml

@@ -174,11 +174,19 @@ spec:
       enabled: true
       v2:
         enabled: true
-
     sidecarInjectorWebhook:
       injectedAnnotations:
         istio.deckhouse.io/version: "{{ $fullVersion }}"
+{{- if ($.Values.global.enabledModules | has "cni-cilium") }}
+      defaultTemplates: ["sidecar", "d8-check-kernel-version"]
+{{- end }}
       templates:
+{{- if ($.Values.global.enabledModules | has "cni-cilium") }}
+        d8-check-kernel-version: |
+          spec:
+            initContainers:
+            {{- include "helm_lib_module_init_container_check_linux_kernel" (tuple $ ">= 5.7") | nindent 12 }}
+{{- end }}
         d8-hold-istio-proxy-termination-until-application-stops: |
           spec:
             containers:

helm_lib/templates/_module_init_container.tpl

@@ -18,3 +18,19 @@
     requests:
       {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 6 }}
 {{- end }}
+
+{{- /* Usage: {{ include "helm_lib_module_init_container_check_linux_kernel" (list . ">= 4.9.17") }} */ -}}
+{{- /* returns initContainer which checks the kernel version on the node for compliance to semver constraint */ -}}
+{{- define "helm_lib_module_init_container_check_linux_kernel"  }}
+  {{- $context := index . 0 -}}
+  {{- $semver_constraint := index . 1  -}}
+- name: check-linux-kernel
+  image: {{ include "helm_lib_module_common_image" (list $context "checkKernelVersion") }}
+  {{- include "helm_lib_module_container_security_context_read_only_root_filesystem" . | nindent 2 }}
+  env:
+  - name: KERNEL_CONSTRAINT
+    value: {{ $semver_constraint | quote }}
+  resources:
+    requests:
+      {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 6 }}
+{{- end }}

modules/000-common/images/check-kernel-version/Dockerfile

@@ -0,0 +1,11 @@
+ARG BASE_SCRATCH
+ARG BASE_GOLANG_17_ALPINE
+FROM $BASE_GOLANG_17_ALPINE as artifact
+WORKDIR /src
+COPY src /src/
+
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o check-kernel-version check-kernel-version.go
+
+FROM $BASE_SCRATCH
+COPY --from=artifact /src/check-kernel-version /
+ENTRYPOINT [ "/check-kernel-version" ]

modules/000-common/images/check-kernel-version/src/check-kernel-version.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2022 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+	"os"
+	"strings"
+
+	"github.com/Masterminds/semver/v3"
+	"golang.org/x/sys/unix"
+)
+
+func main() {
+	kernelConstraint := os.Getenv("KERNEL_CONSTRAINT")
+	if kernelConstraint == "" {
+		log.Fatal("ENV variable KERNEL_CONSTRAINT must be set")
+	}
+	c, err := semver.NewConstraint(kernelConstraint)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	utsname := unix.Utsname{}
+	unix.Uname(&utsname)
+	kernelVersion := string(utsname.Release[:])
+	/* Kernel version should be splitted to parts because versions `5.15.0-52-generic`
+	parses by semver as prerelease version. Prerelease versions by default come before stable versions
+	in the order of precedence, so in semver terms `5.15.0-52-generic` less than `5.15`.
+	More info - https://github.com/Masterminds/semver#working-with-prerelease-versions */
+	v, err := semver.NewVersion(strings.Split(kernelVersion, "-")[0])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if !c.Check(v) {
+		log.Fatalf("the kernel %s does not meet the requirements: %s", kernelVersion, kernelConstraint)
+	}
+	log.Printf("the kernel %s meets the requirements: %s", kernelVersion, kernelConstraint)
+}

modules/000-common/images/check-kernel-version/src/go.mod

@@ -0,0 +1,8 @@
+module check-kernel-version
+
+go 1.18
+
+require (
+	github.com/Masterminds/semver/v3 v3.1.1
+	golang.org/x/sys v0.0.0-20221010170243-090e33056c14
+)

modules/000-common/images/check-kernel-version/src/go.sum

@@ -0,0 +1,4 @@
+github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
+github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

modules/002-deckhouse/hooks/check_kernel_versions.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2022 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+This hook checks nodes kernel requirements and set internal flag stopMainQueue.
+This flag used in another hook, stop_main_queue.go, which stops main queue if flag is true.
+We cannot stop queue in this hook, because we loose metrics if hook fails.
+*/
+
+package hooks
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/flant/addon-operator/pkg/module_manager/go_hook"
+	"github.com/flant/addon-operator/pkg/module_manager/go_hook/metrics"
+	"github.com/flant/addon-operator/sdk"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/deckhouse/deckhouse/go_lib/set"
+)
+
+type nodeKernelVersion struct {
+	Name          string
+	KernelVersion string
+	SemverVersion *semver.Version
+}
+
+type nodeConstraint struct {
+	KernelVersionConstraint string
+	ModulesListInUse        []string
+}
+
+var constraints = []nodeConstraint{
+	{
+		KernelVersionConstraint: ">= 4.9.17",
+		ModulesListInUse:        []string{"cni-cilium"},
+	},
+	{
+		KernelVersionConstraint: ">= 5.7",
+		ModulesListInUse:        []string{"cni-cilium", "istio"},
+	},
+	{
+		KernelVersionConstraint: ">= 5.7",
+		ModulesListInUse:        []string{"cni-cilium", "openvpn"},
+	},
+	{
+		KernelVersionConstraint: ">= 5.7",
+		ModulesListInUse:        []string{"cni-cilium", "node-local-dns"},
+	},
+}
+
+const (
+	nodeKernelCheckMetricsGroup = "node_kernel_check"
+	nodeKernelCheckMetricName   = "d8_node_kernel_does_not_satisfy_requirements"
+)
+
+var _ = sdk.RegisterFunc(&go_hook.HookConfig{
+	Kubernetes: []go_hook.KubernetesConfig{
+		{
+			Name:       "nodes",
+			ApiVersion: "v1",
+			Kind:       "Node",
+			LabelSelector: &metav1.LabelSelector{
+				MatchExpressions: []metav1.LabelSelectorRequirement{
+					{
+						Key:      "node.deckhouse.io/group",
+						Operator: metav1.LabelSelectorOpExists,
+					},
+				},
+			},
+			FilterFunc: filterNodes,
+		},
+	},
+}, handleNodes)
+
+func filterNodes(obj *unstructured.Unstructured) (go_hook.FilterResult, error) {
+	var node corev1.Node
+
+	err := sdk.FromUnstructured(obj, &node)
+	if err != nil {
+		return nil, err
+	}
+	/* Kernel version should be splitted to parts because versions `5.15.0-52-generic`
+	parses by semver as prerelease version. Prerelease versions by default come before stable versions
+	in the order of precedence, so in semver terms `5.15.0-52-generic` less than `5.15`.
+	More info - https://github.com/Masterminds/semver#working-with-prerelease-versions */
+	v, err := semver.NewVersion(strings.Split(node.Status.NodeInfo.KernelVersion, "-")[0])
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse kernel version %s for node %s: %v", node.Status.NodeInfo.KernelVersion, node.Name, err)
+	}
+
+	return nodeKernelVersion{
+		Name:          node.Name,
+		KernelVersion: node.Status.NodeInfo.KernelVersion,
+		SemverVersion: v,
+	}, nil
+}
+
+func handleNodes(input *go_hook.HookInput) error {
+	var hasAffectedNodes bool
+
+	input.MetricsCollector.Expire(nodeKernelCheckMetricsGroup)
+
+	snap := input.Snapshots["nodes"]
+	if len(snap) == 0 {
+		return nil
+	}
+
+	enabledModules := set.NewFromValues(input.Values, "global.enabledModules")
+
+	for _, constrant := range constraints {
+		// check modules in use
+		check := true
+		for _, m := range constrant.ModulesListInUse {
+			if !enabledModules.Has(m) {
+				check = false
+				break
+			}
+		}
+		if !check {
+			continue
+		}
+
+		c, err := semver.NewConstraint(constrant.KernelVersionConstraint)
+		if err != nil {
+			return err
+		}
+
+		for _, n := range snap {
+			node := n.(nodeKernelVersion)
+
+			if !c.Check(node.SemverVersion) {
+				modulesListInUse := strings.Join(constrant.ModulesListInUse, ",")
+				input.MetricsCollector.Set(nodeKernelCheckMetricName, 1, map[string]string{
+					"node":            node.Name,
+					"kernel_version":  node.KernelVersion,
+					"affected_module": modulesListInUse,
+					"constraint":      constrant.KernelVersionConstraint,
+				}, metrics.WithGroup(nodeKernelCheckMetricsGroup))
+				input.LogEntry.Debugf("kernel %s on node %s does not satisfy kernel constraint %s for modules [%s]", node.KernelVersion, node.Name, constrant.KernelVersionConstraint, modulesListInUse)
+				hasAffectedNodes = true
+			}
+		}
+	}
+
+	if hasAffectedNodes {
+		input.LogEntry.Error("some nodes have unmet kernel constraints. To observe affected nodes use the expr `d8_node_kernel_does_not_satisfy_requirements == 1` in Prometheus")
+	}
+
+	return nil
+}