feat(porting/freebsd): make kubelet start & interact with CRI on FreeBSD

This is a vanilla porting making basic parts of the kubelet interact
with FreeBSD and containerd 1.7.0b4 (for now) on FreeBSD

pkg/kubelet/cadvisor/cadvisor_freebsd.go

@@ -0,0 +1,92 @@
+//go:build freebsd
+// +build freebsd
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cadvisor
+
+import (
+	"github.com/google/cadvisor/events"
+	cadvisorapi "github.com/google/cadvisor/info/v1"
+	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	"k8s.io/kubernetes/pkg/kubelet/freebsdstats"
+)
+
+type cadvisorClient struct {
+	rootPath       string
+	bsdStatsClient freebsdstats.Client
+}
+
+var _ Interface = new(cadvisorClient)
+
+// New creates a cAdvisor and exports its API on the specified port if port > 0.
+func New(imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupRoots []string, usingLegacyStats, localStorageCapacityIsolation bool) (Interface, error) {
+	client, err := freebsdstats.NewPerfCounterClient()
+	return &cadvisorClient{
+		rootPath:       rootPath,
+		bsdStatsClient: client,
+	}, err
+}
+
+func (cu *cadvisorClient) Start() error {
+	return nil
+}
+
+func (cu *cadvisorClient) DockerContainer(name string, req *cadvisorapi.ContainerInfoRequest) (cadvisorapi.ContainerInfo, error) {
+	return cadvisorapi.ContainerInfo{}, nil
+}
+
+func (cu *cadvisorClient) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) {
+	return &cadvisorapi.ContainerInfo{}, nil
+}
+
+// ContainerInfoV2 is only expected to be used for the root container. Returns info for all containers in the node.
+func (cu *cadvisorClient) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) {
+	return cu.bsdStatsClient.ContainerInfos()
+}
+
+func (cu *cadvisorClient) GetRequestedContainersInfo(containerName string, options cadvisorapiv2.RequestOptions) (map[string]*cadvisorapi.ContainerInfo, error) {
+	return nil, nil
+}
+
+func (cu *cadvisorClient) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) {
+	return nil, nil
+}
+
+func (cu *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) {
+	return cu.bsdStatsClient.MachineInfo()
+}
+
+func (cu *cadvisorClient) VersionInfo() (*cadvisorapi.VersionInfo, error) {
+	return cu.bsdStatsClient.VersionInfo()
+}
+
+func (cu *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
+	return cadvisorapiv2.FsInfo{}, nil
+}
+
+func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
+	return cu.GetDirFsInfo(cu.rootPath)
+}
+
+func (cu *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) {
+	return &events.EventChannel{}, nil
+}
+
+func (cu *cadvisorClient) GetDirFsInfo(path string) (cadvisorapiv2.FsInfo, error) {
+	return cu.bsdStatsClient.GetDirFsInfo(path)
+}

pkg/kubelet/cadvisor/cadvisor_unsupported.go

@@ -1,5 +1,5 @@
-//go:build !linux && !windows
-// +build !linux,!windows
+//go:build !linux && !windows && !freebsd
+// +build !linux,!windows,!freebsd
 
 /*
 Copyright 2015 The Kubernetes Authors.

pkg/kubelet/cm/container_manager_freebsd.go

@@ -0,0 +1,266 @@
+//go:build freebsd
+// +build freebsd
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// containerManagerImpl implements container manager on FreeBSD.
+
+package cm
+
+import (
+	"fmt"
+
+	"k8s.io/klog/v2"
+	"k8s.io/mount-utils"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/record"
+	internalapi "k8s.io/cri-api/pkg/apis"
+	podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1"
+	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
+	"k8s.io/kubernetes/pkg/kubelet/cm/admission"
+	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
+	"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
+	"k8s.io/kubernetes/pkg/kubelet/cm/dra"
+	"k8s.io/kubernetes/pkg/kubelet/cm/memorymanager"
+	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
+	"k8s.io/kubernetes/pkg/kubelet/config"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+	"k8s.io/kubernetes/pkg/kubelet/pluginmanager/cache"
+	"k8s.io/kubernetes/pkg/kubelet/status"
+	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
+)
+
+type containerManagerImpl struct {
+	// Capacity of this node.
+	capacity v1.ResourceList
+	// Interface for cadvisor.
+	cadvisorInterface cadvisor.Interface
+	// Config of this node.
+	nodeConfig NodeConfig
+	// Interface for exporting and allocating devices reported by device plugins.
+	deviceManager devicemanager.Manager
+	// Interface for Topology resource co-ordination
+	topologyManager topologymanager.Manager
+}
+
+type noopWindowsResourceAllocator struct{}
+
+func (ra *noopWindowsResourceAllocator) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAdmitResult {
+	return admission.GetPodAdmitResult(nil)
+}
+
+func (cm *containerManagerImpl) Start(node *v1.Node,
+	activePods ActivePodsFunc,
+	sourcesReady config.SourcesReady,
+	podStatusProvider status.PodStatusProvider,
+	runtimeService internalapi.RuntimeService,
+	localStorageCapacityIsolation bool) error {
+	klog.V(2).InfoS("Starting Windows container manager")
+
+	if localStorageCapacityIsolation {
+		rootfs, err := cm.cadvisorInterface.RootFsInfo()
+		if err != nil {
+			return fmt.Errorf("failed to get rootfs info: %v", err)
+		}
+		for rName, rCap := range cadvisor.EphemeralStorageCapacityFromFsInfo(rootfs) {
+			cm.capacity[rName] = rCap
+		}
+	}
+
+	// Starts device manager.
+	if err := cm.deviceManager.Start(devicemanager.ActivePodsFunc(activePods), sourcesReady); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// NewContainerManager creates windows container manager.
+func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, nodeConfig NodeConfig, failSwapOn bool, recorder record.EventRecorder, kubeClient clientset.Interface) (ContainerManager, error) {
+	// It is safe to invoke `MachineInfo` on cAdvisor before logically initializing cAdvisor here because
+	// machine info is computed and cached once as part of cAdvisor object creation.
+	// But `RootFsInfo` and `ImagesFsInfo` are not available at this moment so they will be called later during manager starts
+	machineInfo, err := cadvisorInterface.MachineInfo()
+	if err != nil {
+		return nil, err
+	}
+	capacity := cadvisor.CapacityFromMachineInfo(machineInfo)
+
+	cm := &containerManagerImpl{
+		capacity:          capacity,
+		nodeConfig:        nodeConfig,
+		cadvisorInterface: cadvisorInterface,
+	}
+
+	cm.topologyManager = topologymanager.NewFakeManager()
+
+	klog.InfoS("Creating device plugin manager")
+	cm.deviceManager, err = devicemanager.NewManagerImpl(nil, cm.topologyManager)
+	if err != nil {
+		return nil, err
+	}
+	cm.topologyManager.AddHintProvider(cm.deviceManager)
+
+	return cm, nil
+}
+
+func (cm *containerManagerImpl) SystemCgroupsLimit() v1.ResourceList {
+	return v1.ResourceList{}
+}
+
+func (cm *containerManagerImpl) GetNodeConfig() NodeConfig {
+	return NodeConfig{}
+}
+
+func (cm *containerManagerImpl) GetMountedSubsystems() *CgroupSubsystems {
+	return &CgroupSubsystems{}
+}
+
+func (cm *containerManagerImpl) GetQOSContainersInfo() QOSContainersInfo {
+	return QOSContainersInfo{}
+}
+
+func (cm *containerManagerImpl) UpdateQOSCgroups() error {
+	return nil
+}
+
+func (cm *containerManagerImpl) Status() Status {
+	return Status{}
+}
+
+func (cm *containerManagerImpl) GetNodeAllocatableReservation() v1.ResourceList {
+	evictionReservation := hardEvictionReservation(cm.nodeConfig.HardEvictionThresholds, cm.capacity)
+	result := make(v1.ResourceList)
+	for k := range cm.capacity {
+		value := resource.NewQuantity(0, resource.DecimalSI)
+		if cm.nodeConfig.SystemReserved != nil {
+			value.Add(cm.nodeConfig.SystemReserved[k])
+		}
+		if cm.nodeConfig.KubeReserved != nil {
+			value.Add(cm.nodeConfig.KubeReserved[k])
+		}
+		if evictionReservation != nil {
+			value.Add(evictionReservation[k])
+		}
+		if !value.IsZero() {
+			result[k] = *value
+		}
+	}
+	return result
+}
+
+func (cm *containerManagerImpl) GetCapacity(localStorageCapacityIsolation bool) v1.ResourceList {
+	return cm.capacity
+}
+
+func (cm *containerManagerImpl) GetPluginRegistrationHandler() cache.PluginHandler {
+	return cm.deviceManager.GetWatcherHandler()
+}
+
+func (cm *containerManagerImpl) GetDevicePluginResourceCapacity() (v1.ResourceList, v1.ResourceList, []string) {
+	return cm.deviceManager.GetCapacity()
+}
+
+func (cm *containerManagerImpl) NewPodContainerManager() PodContainerManager {
+	return &podContainerManagerStub{}
+}
+
+func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*kubecontainer.RunContainerOptions, error) {
+	opts := &kubecontainer.RunContainerOptions{}
+	// Allocate should already be called during predicateAdmitHandler.Admit(),
+	// just try to fetch device runtime information from cached state here
+	devOpts, err := cm.deviceManager.GetDeviceRunContainerOptions(pod, container)
+	if err != nil {
+		return nil, err
+	} else if devOpts == nil {
+		return opts, nil
+	}
+	opts.Devices = append(opts.Devices, devOpts.Devices...)
+	opts.Mounts = append(opts.Mounts, devOpts.Mounts...)
+	opts.Envs = append(opts.Envs, devOpts.Envs...)
+	opts.Annotations = append(opts.Annotations, devOpts.Annotations...)
+	return opts, nil
+}
+
+func (cm *containerManagerImpl) UpdatePluginResources(node *schedulerframework.NodeInfo, attrs *lifecycle.PodAdmitAttributes) error {
+	return cm.deviceManager.UpdatePluginResources(node, attrs)
+}
+
+func (cm *containerManagerImpl) InternalContainerLifecycle() InternalContainerLifecycle {
+	return &internalContainerLifecycleImpl{cpumanager.NewFakeManager(), memorymanager.NewFakeManager(), topologymanager.NewFakeManager()}
+}
+
+func (cm *containerManagerImpl) GetPodCgroupRoot() string {
+	return ""
+}
+
+func (cm *containerManagerImpl) GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices {
+	return containerDevicesFromResourceDeviceInstances(cm.deviceManager.GetDevices(podUID, containerName))
+}
+
+func (cm *containerManagerImpl) GetAllocatableDevices() []*podresourcesapi.ContainerDevices {
+	return nil
+}
+
+func (cm *containerManagerImpl) ShouldResetExtendedResourceCapacity() bool {
+	return cm.deviceManager.ShouldResetExtendedResourceCapacity()
+}
+
+func (cm *containerManagerImpl) GetAllocateResourcesPodAdmitHandler() lifecycle.PodAdmitHandler {
+	return &noopWindowsResourceAllocator{}
+}
+
+func (cm *containerManagerImpl) UpdateAllocatedDevices() {
+	return
+}
+
+func (cm *containerManagerImpl) GetCPUs(_, _ string) []int64 {
+	return nil
+}
+
+func (cm *containerManagerImpl) GetAllocatableCPUs() []int64 {
+	return nil
+}
+
+func (cm *containerManagerImpl) GetMemory(_, _ string) []*podresourcesapi.ContainerMemory {
+	return nil
+}
+
+func (cm *containerManagerImpl) GetAllocatableMemory() []*podresourcesapi.ContainerMemory {
+	return nil
+}
+
+func (cm *containerManagerImpl) GetNodeAllocatableAbsolute() v1.ResourceList {
+	return nil
+}
+
+func (cm *containerManagerImpl) PrepareDynamicResources(pod *v1.Pod) error {
+	return nil
+}
+
+func (cm *containerManagerImpl) UnprepareDynamicResources(pod *v1.Pod) error {
+	return nil
+}
+
+func (cm *containerManagerImpl) PodMightNeedToUnprepareResources(UID types.UID) bool {
+	return false
+}

pkg/kubelet/cm/container_manager_unsupported.go

@@ -1,5 +1,5 @@
-//go:build !linux && !windows
-// +build !linux,!windows
+//go:build !linux && !windows && !freebsd
+// +build !linux,!windows,!freebsd
 
 /*
 Copyright 2015 The Kubernetes Authors.