Skip to content

Commit

Permalink
[topologymanager] rely on Cadvisor to calculate NUMA distance
Browse files Browse the repository at this point in the history
Signed-off-by: PiotrProkop <pprokop@nvidia.com>
  • Loading branch information
PiotrProkop committed Nov 9, 2022
1 parent 315f0dc commit 540b5bd
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 200 deletions.
51 changes: 3 additions & 48 deletions pkg/kubelet/cm/topologymanager/numa_info.go
Expand Up @@ -18,49 +18,29 @@ package topologymanager

import (
"fmt"
"io/ioutil"
"strconv"
"strings"

cadvisorapi "github.com/google/cadvisor/info/v1"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
)

const (
defaultNodeDir = "/sys/devices/system/node/"
)

type NUMADistances map[int][]uint64

type NUMAInfo struct {
Nodes []int
NUMADistances NUMADistances
}

type NUMASysFs struct {
nodeDir string
}

func NewNUMAInfo(topology []cadvisorapi.Node, opts PolicyOptions) (*NUMAInfo, error) {
return newNUMAInfo(topology, &NUMASysFs{nodeDir: defaultNodeDir}, opts)
}

func newNUMAInfo(topology []cadvisorapi.Node, sysFs *NUMASysFs, opts PolicyOptions) (*NUMAInfo, error) {
var numaNodes []int
distances := map[int][]uint64{}
for _, node := range topology {
numaNodes = append(numaNodes, node.Id)

// Populate the NUMA distances
// For now we need to retrieve this information from sysfs.
// TODO: Update this as follows once a version of cadvisor with this commit is vendored in https://github.com/google/cadvisor/commit/24dd1de08a72cfee661f6178454db995900c0fee
// distances[node.Id] = node.Distances[:]
var nodeDistance []uint64
if opts.PreferClosestNUMA {
var err error
nodeDistance, err = sysFs.GetDistances(node.Id)
if err != nil {
return nil, fmt.Errorf("error getting NUMA distances from sysfs: %w", err)
nodeDistance = node.Distances
if nodeDistance == nil {
return nil, fmt.Errorf("error getting NUMA distances from cadvisor")
}
}
distances[node.Id] = nodeDistance
Expand Down Expand Up @@ -127,28 +107,3 @@ func (d NUMADistances) CalculateAverageFor(bm bitmask.BitMask) float64 {

return sum / count
}

func (s NUMASysFs) GetDistances(nodeId int) ([]uint64, error) {
distancePath := fmt.Sprintf("%s/node%d/distance", s.nodeDir, nodeId)
distance, err := ioutil.ReadFile(distancePath)
if err != nil {
return nil, fmt.Errorf("problem reading %s: %w", distancePath, err)
}

rawDistances := strings.TrimSpace(string(distance))

return splitDistances(rawDistances)
}

func splitDistances(rawDistances string) ([]uint64, error) {
distances := []uint64{}
for _, distance := range strings.Split(rawDistances, " ") {
distanceUint, err := strconv.ParseUint(distance, 10, 64)
if err != nil {
return nil, fmt.Errorf("cannot convert %s to int", distance)
}
distances = append(distances, distanceUint)
}

return distances, nil
}

0 comments on commit 540b5bd

Please sign in to comment.