diff --git a/go.mod b/go.mod index 79f9c9d2..c38a4faf 100644 --- a/go.mod +++ b/go.mod @@ -49,38 +49,38 @@ replace ( google.golang.org/genproto => google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 google.golang.org/grpc => google.golang.org/grpc v1.58.3 gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.2.2 - k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.15-k3s1 - k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.15-k3s1 - k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.15-k3s1 - k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.15-k3s1 - k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.15-k3s1 - k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.15-k3s1 - k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.15-k3s1 - k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.15-k3s1 - k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.15-k3s1 - k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.15-k3s1 - k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.15-k3s1 - k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.15-k3s1 - k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.15-k3s1 - k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.15-k3s1 + k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.16-k3s1 + k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.16-k3s1 + k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.16-k3s1 + k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.16-k3s1 + k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.16-k3s1 + k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.16-k3s1 + k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.16-k3s1 + k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.16-k3s1 + k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.16-k3s1 + k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.16-k3s1 + k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.16-k3s1 + k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.16-k3s1 + k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.16-k3s1 + k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.16-k3s1 k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s2 // k3s-release-1.x k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.60.1-k3s1 // k3s-main - k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.15-k3s1 - k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.15-k3s1 + k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.16-k3s1 + k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.16-k3s1 k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 - k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.25.15-k3s1 - k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.15-k3s1 - k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.15-k3s1 - k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.15-k3s1 - k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.25.15-k3s1 - k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.15-k3s1 - k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.15-k3s1 - k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.15-k3s1 - k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.25.15-k3s1 - k8s.io/pod-security-admission => github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.15-k3s1 - k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.25.15-k3s1 - k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.25.15-k3s1 - k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.25.15-k3s1 + k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.25.16-k3s1 + k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.16-k3s1 + k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.16-k3s1 + k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.16-k3s1 + k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.25.16-k3s1 + k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.16-k3s1 + k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.16-k3s1 + k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.16-k3s1 + k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.25.16-k3s1 + k8s.io/pod-security-admission => github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.16-k3s1 + k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.25.16-k3s1 + k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.25.16-k3s1 + k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.25.16-k3s1 mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 ) @@ -143,15 +143,15 @@ require ( k8s.io/apimachinery v0.27.5 k8s.io/apiserver v0.27.0 k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible - k8s.io/cloud-provider v0.25.15 + k8s.io/cloud-provider v0.25.16 k8s.io/cluster-bootstrap v0.0.0 k8s.io/component-base v0.27.0 k8s.io/component-helpers v0.27.0 k8s.io/cri-api v0.27.5 k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.100.1 - k8s.io/kubectl v0.25.15 - k8s.io/kubernetes v1.25.15 + k8s.io/kubectl v0.25.16 + k8s.io/kubernetes v1.25.16 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index a149aef9..9f76245d 100644 --- a/go.sum +++ b/go.sum @@ -644,56 +644,56 @@ github.com/k3s-io/klog v1.0.0-k3s2 h1:yyvD2bQbxG7m85/pvNctLX2bUDmva5kOBvuZ77tTGB github.com/k3s-io/klog v1.0.0-k3s2/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= github.com/k3s-io/klog/v2 v2.60.1-k3s1 h1:C1hsMF1Eo6heGVQzts6cZ+rDZAReSiOBUxsYMuUkkZI= github.com/k3s-io/klog/v2 v2.60.1-k3s1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -github.com/k3s-io/kubernetes v1.25.15-k3s1 h1:IkNiJleJ8O7b0uhUcLSo6TD4wcTVeWqBCks+sWH8N5g= -github.com/k3s-io/kubernetes v1.25.15-k3s1/go.mod h1:ZrRRMmY0nDNxlnPnJopZRhF2i0tzfB3+DnitJcDe8Vw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.15-k3s1 h1:ZUvIOQa0G9h6dDf4ENxwbuXhatXgtmNV5INzbXfBAI0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.15-k3s1/go.mod h1:/N8bcrUz/rMP42kr63tx/hw8vm8Y7p56XFzZgWfhvMo= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.15-k3s1 h1:0jenTH+slgNz+as9tTE3ElD+aFZU3LSYa43X3Nc6kcQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.15-k3s1/go.mod h1:d/I3BjIbV80Dkus7MugFoUk2WlRSzsfaBGSC+ntsWZg= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.15-k3s1 h1:rwGdQ3loCyWe0Xp9m5khQhzzKD5QcUJR2eQJgWGbSA4= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.15-k3s1/go.mod h1:I5/T97TnTcndA0hLRTsTqeRRQG4RGDk8uZcbs7lMwhE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.15-k3s1 h1:Lg6Z2mzLWj/mf9DjUir1SabUhLaUWuMkmr9n2DywghA= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.15-k3s1/go.mod h1:+ehVFHjV8ewBesDyBWMn21gGuGaP7jms2jvf+aN+B34= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.15-k3s1 h1:xNU2NZrYWe7Y0oGjShSFqZRNlG32P2GlFwfwVYYq7uU= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.15-k3s1/go.mod h1:53ICk6yby1KL6nFqjqxKEBxuGz22fTM8kN50saMpUJs= -github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.15-k3s1 h1:W8WwWLE6FA9ldzyO+zdaXJCkOQw4TSS6mCTRWt4bgz4= -github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.15-k3s1/go.mod h1:iCBSdKXcLFqYDOLMgaZWK0cB3kVp7TCR+eSs56BDUnY= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.15-k3s1 h1:+kBJ6royZr9B6T0KFMdh6eNCNTsKzmBSNKqbsOG7IOw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.15-k3s1/go.mod h1:WKmYlger6OXeolhTZBWytmvy+85J7fdwqFPv2Rfvof8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.15-k3s1 h1:vKYNfo+d2Ut8zbiJpy9YjQTDrhDZ+LeTl9JLQcmyajc= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.15-k3s1/go.mod h1:zrULy04n3XUbx60Nt0SezEPCUCac5azDUAyEBfxZEAQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.15-k3s1 h1:/FjieemXEjHFuEZI3hxmx0yWjmd7WYRNhPgqW6giIAM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.15-k3s1/go.mod h1:HYaqm/S966PX6yJtewIFP0Bd7QQO8UnDGgVu2gT1glM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.15-k3s1 h1:amEe1XBQnQQcbo8miDMrrS2g4+iUgSAdqZLZh2XrQXQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.15-k3s1/go.mod h1:MBxBJEjOchffxgihy+O3qQD3OT9P7mnatA85x5dxO/Q= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.15-k3s1 h1:s4ryne2TPpClKT5CSKcmpg9T+RR7aNu1h0qwtInBeUM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.15-k3s1/go.mod h1:fo9V6qV3s0BXcgdJV3wmisFS3H3DnsSfpjWrfGvPUaE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.15-k3s1 h1:zVU2maYLxOmz++Z2gvD3W8wsuiqFUaRkUB4pVrF3Dy0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.15-k3s1/go.mod h1:PeQzOnq8NDoaP/UG7OTmVW5xJabzhUCeAipkAsH0rfI= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.15-k3s1 h1:OvxK4fgBomcSNuXptJRAuxogeiMJPiK2Ki19xcvBi4k= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.15-k3s1/go.mod h1:fjRHAkEDvhp4CYP+uH2fwDaVphp7CvNqBwMK4W6u3CI= -github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.15-k3s1 h1:Wcb1DDfe+23rRpT+ZKtILBsnCPTSLwjYqqqooodNJLk= -github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.15-k3s1/go.mod h1:IEPPKueX822CfmQMaPJIdZkIN3sVmJMseMV5qee067I= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.15-k3s1 h1:i9Qlg9y9YPHMStEe2GrYuE9qU3FcLc1/lKQ2gXOWFW8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.15-k3s1/go.mod h1:M+yHpazu/Q7h/sDtYnTd0tPlx/oIooBYcUWLOlSjylE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.15-k3s1 h1:eIDGoamtKtWiUhf/xIHOJofxvmrqcFJBbNHUjzzUfHg= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.15-k3s1/go.mod h1:fw3oJiD0qKJSk5X3CYET9Y0ca5/2kaNhzcyWAC/x1YE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.25.15-k3s1/go.mod h1:2c4F3OIIqIDXcNR/DiI2Wz56nlLh7lKd9ofA0m4BZ9c= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.15-k3s1 h1:cy80/faKaLvdeJkQeQ6ZaJJCWWWX2ZgiSkZcUQ2CqNM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.15-k3s1/go.mod h1:Hlpchmj7z0fZqDm4H391IfBtQ0gqJTp2aaDlV6CIf8s= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.15-k3s1 h1:MRvVWMgDH7IxYAjYOCu99by3Lq0qs+zYSj1tcITvEEw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.15-k3s1/go.mod h1:i6Ffq/3boXLMwfDKYNBKRvom1HRthHjSskxV61bFuPE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.15-k3s1 h1:Jka+LYT1P6PWE9SC1UO4j2o6LUaDDsSWV9Z30d5gkQw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.15-k3s1/go.mod h1:49Qgz77HFyZkojPrPgKF3QcCpDEk88NeCRwi/0i+QhQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.15-k3s1 h1:Rf/TEWFEBZB9OjpiVHyg62W3ugGFrJ+pMnkSMcg+p8A= -github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.15-k3s1/go.mod h1:fB49PcIyv3A8Bg+xOQqTf7FOxCUA8C6BGGcQIwAoiu4= -github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.15-k3s1 h1:7kD7mSuczGyVurGP2gw06IIf08bQ2hYhvUo/aF3zVTA= -github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.15-k3s1/go.mod h1:tELkwF9xCmwJ3l3siCo2niUaSIyjTpZ+64S+Z8fTcUw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.15-k3s1 h1:RHmLHwd2CJWCalJH24zRVcvSQakb/UjWe0LkzfdXTH8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.15-k3s1/go.mod h1:Mjv5B8KCT278Nm2okrd1rijnIpNP9Oys7TIkqGF1qHY= -github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.15-k3s1 h1:qJTPqJTSY3qZ6XrueC7w1A4J+QcPeQMNEmBLANppMB8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.15-k3s1/go.mod h1:Sa550rNU+kF1UB09tYc4SyKC73NCcD0vEYR+JCt6IO0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.25.15-k3s1/go.mod h1:wgON9dzQlkboLpD5iQw/jJhYh4tyI9RLBo/XHZlGlKk= +github.com/k3s-io/kubernetes v1.25.16-k3s1 h1:PFCNVUmSSH1qyUdGeBCMd8LkS7fgVIc0iCcM/8hjt5o= +github.com/k3s-io/kubernetes v1.25.16-k3s1/go.mod h1:ZrRRMmY0nDNxlnPnJopZRhF2i0tzfB3+DnitJcDe8Vw= +github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.16-k3s1 h1:9aafKwK0KOzOvBabLvHlzS39AfVIMQZXHSIzpGLtg14= +github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.16-k3s1/go.mod h1:/N8bcrUz/rMP42kr63tx/hw8vm8Y7p56XFzZgWfhvMo= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.16-k3s1 h1:mtA75stK9PBwwXpDYPiwog7/3h2AgbhNUGbRFEJGFEM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.16-k3s1/go.mod h1:d/I3BjIbV80Dkus7MugFoUk2WlRSzsfaBGSC+ntsWZg= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.16-k3s1 h1:cdDj1YvlcOXsuEa7+O58KxOUT6RAW5il+iLJHF1pTKg= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.16-k3s1/go.mod h1:I5/T97TnTcndA0hLRTsTqeRRQG4RGDk8uZcbs7lMwhE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.16-k3s1 h1:IoGv7iGxAqeEAdmLSjPqOvhnMrhdbnzgmxUxczQebRs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.16-k3s1/go.mod h1:+ehVFHjV8ewBesDyBWMn21gGuGaP7jms2jvf+aN+B34= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.16-k3s1 h1:hmjTkWREnkmhrczbT9KTkorDvufWdYuAAHYklLaDnCA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.16-k3s1/go.mod h1:53ICk6yby1KL6nFqjqxKEBxuGz22fTM8kN50saMpUJs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.16-k3s1 h1:VSXpJrPdOZmNKl4fLid69o/sSeet+wQdEZvgB+MOFrM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.16-k3s1/go.mod h1:iCBSdKXcLFqYDOLMgaZWK0cB3kVp7TCR+eSs56BDUnY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.16-k3s1 h1:TkhcdxWmseTVLEZ4NvDLBAVgerjdHzj1GnvSPxv6GIA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.16-k3s1/go.mod h1:WKmYlger6OXeolhTZBWytmvy+85J7fdwqFPv2Rfvof8= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.16-k3s1 h1:p0x2Hh8CAblmMLGTirLAGfTNkQCGTiZDI/Sykvu+wK4= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.16-k3s1/go.mod h1:zrULy04n3XUbx60Nt0SezEPCUCac5azDUAyEBfxZEAQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.16-k3s1 h1:fGV5gLJwSUCD6+Y/pnbSjniho6bLMKSNeViqEYqX8Hs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.16-k3s1/go.mod h1:HYaqm/S966PX6yJtewIFP0Bd7QQO8UnDGgVu2gT1glM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.16-k3s1 h1:o8RWF/WKEByECD023IGeeS/yT0ifeThQjYJVEhQZvB8= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.16-k3s1/go.mod h1:MBxBJEjOchffxgihy+O3qQD3OT9P7mnatA85x5dxO/Q= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.16-k3s1 h1:MhiL58ayWui5dLxz2laRm00MEEUEOxfMKgGKtFcIWKc= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.16-k3s1/go.mod h1:fo9V6qV3s0BXcgdJV3wmisFS3H3DnsSfpjWrfGvPUaE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.16-k3s1 h1:i7X/tZQQzr7Jv5G8m5GbTQm7QmstPQzXbW5FzHVJNOI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.16-k3s1/go.mod h1:PeQzOnq8NDoaP/UG7OTmVW5xJabzhUCeAipkAsH0rfI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.16-k3s1 h1:otmcf5mYOGXMoBTIyL9QcPToYSDtp7WYvbRHCp2k3MM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.16-k3s1/go.mod h1:fjRHAkEDvhp4CYP+uH2fwDaVphp7CvNqBwMK4W6u3CI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.16-k3s1 h1:eqzYjx6OoJuu3nbZC0JDo/dTtUDIMZdyWFquudRPG9s= +github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.16-k3s1/go.mod h1:IEPPKueX822CfmQMaPJIdZkIN3sVmJMseMV5qee067I= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.16-k3s1 h1:bV5OwxCITt/5kpVx7dfyn7Vq+/9MSCScgxZCKR5PiOY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.16-k3s1/go.mod h1:M+yHpazu/Q7h/sDtYnTd0tPlx/oIooBYcUWLOlSjylE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.16-k3s1 h1:Vcj7swoM36pf45NGBkEIUxdLcGaiMVTZTqAYCzrtVD4= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.16-k3s1/go.mod h1:fw3oJiD0qKJSk5X3CYET9Y0ca5/2kaNhzcyWAC/x1YE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.25.16-k3s1/go.mod h1:2c4F3OIIqIDXcNR/DiI2Wz56nlLh7lKd9ofA0m4BZ9c= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.16-k3s1 h1:uxr/r3pHs0spLIMT5MKyKhuVMJioCiLI4TPGk8qxhbQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.16-k3s1/go.mod h1:Hlpchmj7z0fZqDm4H391IfBtQ0gqJTp2aaDlV6CIf8s= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.16-k3s1 h1:ag149duTKGYxdPUe/NbdSzE6qDfkBLGTGSI6UEWzOpA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.16-k3s1/go.mod h1:i6Ffq/3boXLMwfDKYNBKRvom1HRthHjSskxV61bFuPE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.16-k3s1 h1:EXvl/CSg3s6kKyBwJQOY6qLKuuAAoODoVeWFhAIWfCM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.16-k3s1/go.mod h1:49Qgz77HFyZkojPrPgKF3QcCpDEk88NeCRwi/0i+QhQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.16-k3s1 h1:eM1M/I0/L8zXidLag6s2oHuSk8duaATXQ0mvJc5MMe0= +github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.16-k3s1/go.mod h1:fB49PcIyv3A8Bg+xOQqTf7FOxCUA8C6BGGcQIwAoiu4= +github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.16-k3s1 h1:DSfgxUnWyEZFWrbvY1vMX9JxI8PB5CHLdXCP2O8BM7A= +github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.16-k3s1/go.mod h1:tELkwF9xCmwJ3l3siCo2niUaSIyjTpZ+64S+Z8fTcUw= +github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.16-k3s1 h1:GTfnOebp9SqM3tgjgIulPYKUtx+3JwzXW+3BlDnAAPQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.16-k3s1/go.mod h1:Mjv5B8KCT278Nm2okrd1rijnIpNP9Oys7TIkqGF1qHY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.16-k3s1 h1:56yP2tf+ckLaou4sHZcDrrAcNCRJk271vftvl6Lanto= +github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.16-k3s1/go.mod h1:Sa550rNU+kF1UB09tYc4SyKC73NCcD0vEYR+JCt6IO0= +github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.25.16-k3s1/go.mod h1:wgON9dzQlkboLpD5iQw/jJhYh4tyI9RLBo/XHZlGlKk= github.com/k3s-io/stargz-snapshotter v0.14.3-k3s1 h1:29UZBqL+9Kcwl7ES7oUdN9UtS0oVZY6AL+IujwGNwPk= github.com/k3s-io/stargz-snapshotter v0.14.3-k3s1/go.mod h1:j2Ya4JeA5gMZJr8BchSkPjlcCEh++auAxp4nidPI6N0= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= diff --git a/manifests/runtimes.yaml b/manifests/runtimes.yaml new file mode 100644 index 00000000..fdbc9512 --- /dev/null +++ b/manifests/runtimes.yaml @@ -0,0 +1,59 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia-experimental +handler: nvidia-experimental +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: crun +handler: crun +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: lunatic +handler: lunatic +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: slight +handler: slight +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: spin +handler: spin +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wws +handler: wws +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmedge +handler: wasmedge +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmer +handler: wasmer +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmtime +handler: wasmtime \ No newline at end of file diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index 81666437..96f6f5e2 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -57,6 +57,13 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { cfg.AgentConfig.Systemd = !isRunningInUserNS && controllers["cpuset"] && os.Getenv("INVOCATION_ID") != "" } + extraRuntimes := findContainerRuntimes(os.DirFS(string(os.PathSeparator))) + + // Verifies if the DefaultRuntime can be found + if _, ok := extraRuntimes[cfg.DefaultRuntime]; !ok && cfg.DefaultRuntime != "" { + return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime) + } + var containerdTemplate string containerdConfig := templates.ContainerdConfig{ NodeConfig: cfg, @@ -65,7 +72,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { IsRunningInUserNS: isRunningInUserNS, EnableUnprivileged: kernel.CheckKernelVersion(4, 11, 0), PrivateRegistryConfig: privRegistries.Registry, - ExtraRuntimes: findNvidiaContainerRuntimes(os.DirFS(string(os.PathSeparator))), + ExtraRuntimes: extraRuntimes, Program: version.Program, } diff --git a/pkg/agent/containerd/nvidia.go b/pkg/agent/containerd/nvidia.go deleted file mode 100644 index 89a2f06b..00000000 --- a/pkg/agent/containerd/nvidia.go +++ /dev/null @@ -1,67 +0,0 @@ -//go:build linux -// +build linux - -package containerd - -import ( - "errors" - "io/fs" - "path/filepath" - - "github.com/sirupsen/logrus" - "github.com/xiaods/k8e/pkg/agent/templates" -) - -// findNvidiaContainerRuntimes returns a list of nvidia container runtimes that -// are available on the system. It checks install locations used by the nvidia -// gpu operator and by system package managers. The gpu operator installation -// takes precedence over the system package manager installation. -// The given fs.FS should represent the filesystem root directory to search in. -func findNvidiaContainerRuntimes(root fs.FS) map[string]templates.ContainerdRuntimeConfig { - // Check these locations in order. The GPU operator's installation should - // take precedence over the package manager's installation. - locationsToCheck := []string{ - "usr/local/nvidia/toolkit", // Path when installing via GPU Operator - "usr/bin", // Path when installing via package manager - } - - // Fill in the binary location with just the name of the binary, - // and check against each of the possible locations. If a match is found, - // set the location to the full path. - potentialRuntimes := map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "nvidia-container-runtime-experimental", - }, - } - foundRuntimes := map[string]templates.ContainerdRuntimeConfig{} -RUNTIME: - for runtimeName, runtimeConfig := range potentialRuntimes { - for _, location := range locationsToCheck { - binaryPath := filepath.Join(location, runtimeConfig.BinaryName) - logrus.Debugf("Searching for %s container runtime at /%s", runtimeName, binaryPath) - if info, err := fs.Stat(root, binaryPath); err == nil { - if info.IsDir() { - logrus.Debugf("Found %s container runtime at /%s, but it is a directory. Skipping.", runtimeName, binaryPath) - continue - } - runtimeConfig.BinaryName = filepath.Join("/", binaryPath) - logrus.Infof("Found %s container runtime at %s", runtimeName, runtimeConfig.BinaryName) - foundRuntimes[runtimeName] = runtimeConfig - // Skip to the next runtime to enforce precedence. - continue RUNTIME - } else { - if errors.Is(err, fs.ErrNotExist) { - logrus.Debugf("%s container runtime not found at /%s", runtimeName, binaryPath) - } else { - logrus.Errorf("Error searching for %s container runtime at /%s: %v", runtimeName, binaryPath, err) - } - } - } - } - return foundRuntimes -} diff --git a/pkg/agent/containerd/nvidia_test.go b/pkg/agent/containerd/nvidia_test.go deleted file mode 100644 index fc0a9900..00000000 --- a/pkg/agent/containerd/nvidia_test.go +++ /dev/null @@ -1,219 +0,0 @@ -//go:build linux -// +build linux - -package containerd - -import ( - "io/fs" - "reflect" - "testing" - "testing/fstest" - - "github.com/xiaods/k8e/pkg/agent/templates" -) - -func Test_UnitFindNvidiaContainerRuntimes(t *testing.T) { - executable := &fstest.MapFile{Mode: 0755} - type args struct { - root fs.FS - } - tests := []struct { - name string - args args - want map[string]templates.ContainerdRuntimeConfig - }{ - { - name: "No runtimes", - args: args{ - root: fstest.MapFS{}, - }, - want: map[string]templates.ContainerdRuntimeConfig{}, - }, - { - name: "Nvidia runtime in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - }, - }, - { - name: "Experimental runtime in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - }, - }, - { - name: "Two runtimes in separate directories", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - }, - }, - { - name: "Experimental runtime in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Same runtime in two directories", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/bin/nvidia-container-runtime": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in both directories", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - "usr/bin/nvidia-container-runtime": executable, - "usr/bin/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/bin and one duplicate in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Runtime is a directory", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": &fstest.MapFile{ - Mode: fs.ModeDir, - }, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{}, - }, - { - name: "Runtime in both directories, but one is a directory", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ - Mode: fs.ModeDir, - }, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := findNvidiaContainerRuntimes(tt.args.root); !reflect.DeepEqual(got, tt.want) { - t.Errorf("findNvidiaContainerRuntimes() = %+v\nWant = %+v", got, tt.want) - } - }) - } -} diff --git a/pkg/agent/containerd/runtimes.go b/pkg/agent/containerd/runtimes.go new file mode 100644 index 00000000..df056c82 --- /dev/null +++ b/pkg/agent/containerd/runtimes.go @@ -0,0 +1,161 @@ +//go:build linux +// +build linux + +package containerd + +import ( + "errors" + "io/fs" + "path/filepath" + + "github.com/sirupsen/logrus" + "github.com/xiaods/k8e/pkg/agent/templates" +) + +// A map with string as value and `templates.ContainerdRuntimeConfig` as values. +// The key holds the name of the runtime +type runtimeConfigs map[string]templates.ContainerdRuntimeConfig + +// searchForRuntimes searches for runtimes and add into foundRuntimes +// It checks install locations provided via potentitalRuntimes variable. +// The binaries are searched at the locations specivied by locationsToCheck. +// The given fs.FS should represent the filesystem root directory to search in. +func searchForRuntimes(root fs.FS, potentialRuntimes runtimeConfigs, locationsToCheck []string, foundRuntimes runtimeConfigs) { + // Check these locations in order. The GPU operator's installation should + // take precedence over the package manager's installation. + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + for runtimeName, runtimeConfig := range potentialRuntimes { + for _, location := range locationsToCheck { + binaryPath := filepath.Join(location, runtimeConfig.BinaryName) + logrus.Debugf("Searching for %s container runtime at /%s", runtimeName, binaryPath) + if info, err := fs.Stat(root, binaryPath); err == nil { + if info.IsDir() { + logrus.Debugf("Found %s container runtime at /%s, but it is a directory. Skipping.", runtimeName, binaryPath) + continue + } + runtimeConfig.BinaryName = filepath.Join("/", binaryPath) + logrus.Infof("Found %s container runtime at %s", runtimeName, runtimeConfig.BinaryName) + foundRuntimes[runtimeName] = runtimeConfig + break + } else { + if errors.Is(err, fs.ErrNotExist) { + logrus.Debugf("%s container runtime not found at /%s", runtimeName, binaryPath) + } else { + logrus.Errorf("Error searching for %s container runtime at /%s: %v", runtimeName, binaryPath, err) + } + } + } + } +} + +// findContainerRuntimes is a function that searches for all the runtimes and +// return a list with all the runtimes that have been found +func findContainerRuntimes(root fs.FS) runtimeConfigs { + foundRuntimes := runtimeConfigs{} + findCRunContainerRuntime(root, foundRuntimes) + findNvidiaContainerRuntimes(root, foundRuntimes) + findWasiRuntimes(root, foundRuntimes) + return foundRuntimes +} + +// findCRunContainerRuntime finds if crun is available in the system and adds to foundRuntimes +func findCRunContainerRuntime(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. + locationsToCheck := []string{ + "usr/sbin", // Path when installing via package manager + "usr/bin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "crun": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "crun", + }, + } + + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} + +// findNvidiaContainerRuntimes finds the nvidia runtimes that are are available on the system +// and adds to foundRuntimes. It checks install locations used by the nvidia +// gpu operator and by system package managers. The gpu operator installation +// takes precedence over the system package manager installation. +// The given fs.FS should represent the filesystem root directory to search in. +func findNvidiaContainerRuntimes(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. The GPU operator's installation should + // take precedence over the package manager's installation. + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path when installing via GPU Operator + "usr/bin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime-experimental", + }, + } + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} + +// findWasiRuntimes finds the WebAssembly (WASI) container runtimes that +// are available on the system and adds to foundRuntimes. It checks install locations used by the kwasm +// operator and by system package managers. The kwasm operator installation +// takes precedence over the system package manager installation. +// The given fs.FS should represent the filesystem root directory to search in. +func findWasiRuntimes(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. + locationsToCheck := []string{ + "opt/kwasm/bin", // Path when installing via kwasm Operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "containerd-shim-wasmer-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + } + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} diff --git a/pkg/agent/containerd/runtimes_test.go b/pkg/agent/containerd/runtimes_test.go new file mode 100644 index 00000000..b209b495 --- /dev/null +++ b/pkg/agent/containerd/runtimes_test.go @@ -0,0 +1,796 @@ +//go:build linux +// +build linux + +package containerd + +import ( + "io/fs" + "reflect" + "testing" + "testing/fstest" +) + +func Test_UnitFindContainerRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + + type args struct { + root fs.FS + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + }, + want: runtimeConfigs{}, + }, + { + name: "Found crun, nvidia and wasm", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/crun": executable, + "opt/kwasm/bin/containerd-shim-lunatic-v1": executable, + }, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "crun": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/crun", + }, + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-lunatic-v1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := findContainerRuntimes(tt.args.root) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("findContainerRuntimes = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} + +func Test_UnitSearchContainerRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator + "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + potentialRuntimes := runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "containerd-shim-spin-v1", + }, + } + + type args struct { + root fs.FS + potentialRuntimes runtimeConfigs + locationsToCheck []string + } + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{}, + }, + { + name: "Nvidia runtime in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + { + name: "Two runtimes in separate directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "opt/kwasm/bin/containerd-shim-spin-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-spin-v1", + }, + }, + }, + { + name: "Same runtime in two directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-spin-v1": executable, + "opt/kwasm/bin/containerd-shim-spin-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-spin-v1", + }, + }, + }, + { + name: "Both runtimes in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-spin-v1": executable, + "usr/bin/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/usr/bin/containerd-shim-spin-v1", + }, + }, + }, + { + name: "Both runtimes in both directories", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/containerd-shim-spin-v1": executable, + "opt/kwasm/bin/containerd-shim-spin-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-spin-v1", + }, + }, + }, + { + name: "Both runtimes in /usr/bin and one duplicate in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/containerd-shim-spin-v1": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/usr/bin/containerd-shim-spin-v1", + }, + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + }, + }, + { + name: "Runtime is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{}, + }, + { + name: "Runtime in both directories, but one is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := runtimeConfigs{} + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("findContainerRuntimes() = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} + +func Test_UnitSearchWasiRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator + "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + potentialRuntimes := runtimeConfigs{ + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "containerd-shim-wasmer-v1", + }, + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + } + + type args struct { + root fs.FS + potentialRuntimes runtimeConfigs + locationsToCheck []string + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{}, + }, + { + name: "wasmtime runtime in /usr/sbin", + args: args{ + root: fstest.MapFS{ + "usr/sbin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/usr/sbin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "lunatic runtime in /opt/kwasm/bin/", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-lunatic-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-lunatic-v1", + }, + }, + }, + { + name: "Two runtimes in separate directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-wasmer-v1": executable, + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "/usr/bin/containerd-shim-wasmer-v1", + }, + }, + }, + { + name: "Same runtime in two directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-wasmedge-v1": executable, + "opt/kwasm/bin/containerd-shim-wasmedge-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmedge-v1", + }, + }, + }, + { + name: "All runtimes in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-lunatic-v1": executable, + "usr/bin/containerd-shim-slight-v1": executable, + "usr/bin/containerd-shim-spin-v1": executable, + "usr/bin/containerd-shim-wws-v1": executable, + "usr/bin/containerd-shim-wasmedge-v1": executable, + "usr/bin/containerd-shim-wasmer-v1": executable, + "usr/bin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/usr/bin/containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/usr/bin/containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/usr/bin/containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "/usr/bin/containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "/usr/bin/containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "/usr/bin/containerd-shim-wasmer-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/usr/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "Both runtimes in both directories", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "usr/bin/containerd-shim-slight-v1": executable, + "usr/bin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "Preserve already found runtimes", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "usr/bin/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := runtimeConfigs{} + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("searchForRuntimes = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} + +func Test_UnitSearchNvidiaContainerRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator + "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + potentialRuntimes := runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime-experimental", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + } + + type args struct { + root fs.FS + potentialRuntimes runtimeConfigs + locationsToCheck []string + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{}, + }, + { + name: "Nvidia runtime in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + { + name: "Experimental runtime in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + }, + }, + { + name: "Two runtimes in separate directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + }, + }, + { + name: "Experimental runtime in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Same runtime in two directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/bin/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in both directories", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/bin and one duplicate in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Runtime is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{}, + }, + { + name: "Runtime in both directories, but one is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + { + name: "Preserve already found runtimes", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := runtimeConfigs{} + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("searchForRuntimes() = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} diff --git a/pkg/agent/run_linux.go b/pkg/agent/run_linux.go index 3dc8016e..fa203dc2 100644 --- a/pkg/agent/run_linux.go +++ b/pkg/agent/run_linux.go @@ -36,6 +36,11 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error { } } + // Send to node struct the value from cli/config default runtime + if cfg.DefaultRuntime != "" { + nodeConfig.DefaultRuntime = cfg.DefaultRuntime + } + crp := "runtime-endpoint: " + cre + "\n" ise := nodeConfig.ImageServiceEndpoint if ise != "" && ise != cre { diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 43cf65f2..2a8e51df 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -32,6 +32,7 @@ version = 2 [plugins."io.containerd.grpc.v1.cri".containerd] snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} + {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}} {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} {{ if .NodeConfig.AgentConfig.ImageServiceSocket }} [plugins."io.containerd.snapshotter.v1.stargz"] diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index eb8f5b4a..c9bc7a1c 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -27,6 +27,7 @@ type Agent struct { Snapshotter string Docker bool ContainerRuntimeEndpoint string + DefaultRuntime string ImageServiceEndpoint string Debug bool Rootless bool @@ -109,6 +110,11 @@ var ( Usage: "(agent/runtime) Disable embedded containerd and use the CRI socket at the given path; when used with --docker this sets the docker socket path", Destination: &AgentConfig.ContainerRuntimeEndpoint, } + DefaultRuntimeFlag = &cli.StringFlag{ + Name: "default-runtime", + Usage: "(agent/runtime) Set the default runtime in containerd", + Destination: &AgentConfig.DefaultRuntime, + } ImageServiceEndpointFlag = &cli.StringFlag{ Name: "image-service-endpoint", Usage: "(agent/runtime) Disable embedded containerd image service and use remote image service socket at the given path. If not specified, defaults to --container-runtime-endpoint.", @@ -236,6 +242,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { LBServerPortFlag, ProtectKernelDefaultsFlag, CRIEndpointFlag, + DefaultRuntimeFlag, ImageServiceEndpointFlag, PauseImageFlag, SnapshotterFlag, diff --git a/pkg/cli/cmds/stage.go b/pkg/cli/cmds/stage.go index a5172f65..daef2e75 100644 --- a/pkg/cli/cmds/stage.go +++ b/pkg/cli/cmds/stage.go @@ -7,5 +7,5 @@ const ( // coredns run controllers that are turned off when their manifests are disabled. // The k8e CloudController also has a bundled manifest and can be disabled via the // --disable-cloud-controller flag or --disable=ccm, but the latter method is not documented. - DisableItems = "coredns, local-storage, metrics-server" + DisableItems = "coredns, local-storage, metrics-server, runtimes" ) diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 457ce710..63558c37 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -42,6 +42,7 @@ type Node struct { Token string Certificate *tls.Certificate ServerHTTPSPort int + DefaultRuntime string } type Containerd struct { diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index 3a4ae6f3..77881f9e 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -12,6 +12,7 @@ // manifests/metrics-server/metrics-server-service.yaml // manifests/metrics-server/resource-reader.yaml // manifests/rolebindings.yaml +// manifests/runtimes.yaml //go:build !no_stage // +build !no_stage @@ -331,6 +332,26 @@ func rolebindingsYaml() (*asset, error) { return a, nil } +var _runtimesYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\xd0\x31\x8e\x84\x30\x0c\x05\xd0\x3e\xa7\xc8\x05\xc2\x6a\xbb\x55\xda\xbd\xc1\x14\xd3\x5b\xc4\x02\x8b\xc4\xa0\x38\xc0\x1c\x7f\x04\x1a\x26\x40\xed\xf2\x7f\x4b\xef\x4b\x86\x89\x9e\x98\x85\x46\xf6\x96\xc7\x80\xcd\xf0\x27\x0d\x8d\x3f\xcb\xaf\x19\x88\x83\xb7\x8f\x99\x0b\x25\xfc\x8f\x20\x62\x12\x16\x08\x50\xc0\x1b\x6b\x19\x12\x7a\xcb\x0b\x05\x02\xd3\x03\x87\x88\xf9\x9b\x9d\x73\x46\x87\x76\xf8\x9a\x30\x53\x42\x2e\x10\xef\x3b\xd7\xa3\xc6\x68\x9b\x67\xae\x2b\x7b\xd2\x60\xe3\xcc\x50\xa8\xad\xf2\x51\x68\xe0\x12\xa9\xeb\x4b\xb5\x3f\x59\x85\x9e\xe8\xf4\x8e\x3d\x69\xb0\xeb\x2a\x55\xdd\x82\x0a\x0a\x92\x30\x74\x78\x92\x8f\x46\x8d\xcf\x37\x3c\xab\xd1\xdb\xfd\x8a\x6f\xcd\x3b\x00\x00\xff\xff\xc7\xad\x48\x21\x9f\x03\x00\x00") + +func runtimesYamlBytes() ([]byte, error) { + return bindataRead( + _runtimesYaml, + "runtimes.yaml", + ) +} + +func runtimesYaml() (*asset, error) { + bytes, err := runtimesYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "runtimes.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + // Asset loads and returns the asset for the given name. // It returns an error if the asset could not be found or // could not be loaded. @@ -395,6 +416,7 @@ var _bindata = map[string]func() (*asset, error){ "metrics-server/metrics-server-service.yaml": metricsServerMetricsServerServiceYaml, "metrics-server/resource-reader.yaml": metricsServerResourceReaderYaml, "rolebindings.yaml": rolebindingsYaml, + "runtimes.yaml": runtimesYaml, } // AssetDir returns the file names below a certain @@ -454,6 +476,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "resource-reader.yaml": &bintree{metricsServerResourceReaderYaml, map[string]*bintree{}}, }}, "rolebindings.yaml": &bintree{rolebindingsYaml, map[string]*bintree{}}, + "runtimes.yaml": &bintree{runtimesYaml, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index bc3c24e3..6cd8ddbf 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -178,7 +178,7 @@ func (s *S3) upload(ctx context.Context, snapshot string, extraMetadata *v1.Conf if _, err := s.uploadSnapshotMetadata(ctx, metadataKey, metadata); err != nil { logrus.Warnf("Failed to upload snapshot metadata to S3: %v", err) } else { - logrus.Infof("Uploaded snapshot metadata s3://%s/%s", s.config.EtcdS3BucketName, metadata) + logrus.Infof("Uploaded snapshot metadata s3://%s/%s", s.config.EtcdS3BucketName, metadataKey) } return sf, err } @@ -358,9 +358,8 @@ func (s *S3) listSnapshots(ctx context.Context) (map[string]snapshotFile, error) defer cancel() opts := minio.ListObjectsOptions{ - Prefix: s.config.EtcdS3Folder, - Recursive: true, - WithMetadata: true, + Prefix: s.config.EtcdS3Folder, + Recursive: true, } objects := s.client.ListObjects(ctx, s.config.EtcdS3BucketName, opts) diff --git a/pkg/etcd/snapshot.go b/pkg/etcd/snapshot.go index 98f3bf8f..a99dba48 100644 --- a/pkg/etcd/snapshot.go +++ b/pkg/etcd/snapshot.go @@ -368,14 +368,16 @@ func (e *ETCD) Snapshot(ctx context.Context) error { } else { logrus.Infof("S3 upload complete for %s", snapshotName) } + // Attempt to apply retention even if the upload failed; failure may be due to bucket + // being full or some other condition that retention policy would resolve. + if err := e.s3.snapshotRetention(ctx); err != nil { + logrus.Errorf("Failed to apply s3 snapshot retention policy: %v", err) + } } + // sf is either s3 snapshot metadata, or s3 failure record if err := e.addSnapshotData(*sf); err != nil { return errors.Wrap(err, "failed to sync ETCDSnapshotFile") } - if err := e.s3.snapshotRetention(ctx); err != nil { - logrus.Errorf("Failed to apply s3 snapshot retention policy: %v", err) - } - } } @@ -547,6 +549,7 @@ func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error { } if e.config.EtcdS3 { if err := e.initS3IfNil(ctx); err != nil { + logrus.Warnf("Unable to initialize S3 client: %v", err) return err } } @@ -740,6 +743,7 @@ func (e *ETCD) ReconcileSnapshotData(ctx context.Context) error { // Get snapshots from S3 if e.config.EtcdS3 { if err := e.initS3IfNil(ctx); err != nil { + logrus.Warnf("Unable to initialize S3 client: %v", err) return err }