-
Notifications
You must be signed in to change notification settings - Fork 103
/
topology.go
525 lines (449 loc) · 13.7 KB
/
topology.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
/*
Copyright 2022 The Katalyst Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package machine
import (
"fmt"
info "github.com/google/cadvisor/info/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
)
// NUMANodeInfo is a map from NUMANode ID to a list of
// CPU IDs associated with that NUMANode.
type NUMANodeInfo map[int]CPUSet
// CPUDetails is a map from CPU ID to Core ID, Socket ID, and NUMA ID.
type CPUDetails map[int]CPUInfo
// CPUTopology contains details of node cpu, where :
// CPU - logical CPU, cadvisor - thread
// Core - physical CPU, cadvisor - Core
// Socket - socket, cadvisor - Socket
// NUMA Node - NUMA cell, cadvisor - Node
type CPUTopology struct {
NumCPUs int
NumCores int
NumSockets int
NumNUMANodes int
CPUDetails CPUDetails
}
type MemoryDetails map[int]uint64
// Equal returns true if the MemoryDetails map is equal to the supplied MemoryDetails
func (d MemoryDetails) Equal(want MemoryDetails) bool {
if len(d) != len(want) {
return false
}
for k, v := range d {
if v != want[k] {
return false
}
}
return true
}
// Clone creates a new MemoryDetails instance with the same content.
func (d MemoryDetails) Clone() MemoryDetails {
if d == nil {
return nil
}
clone := make(MemoryDetails)
for key, value := range d {
clone[key] = value
}
return clone
}
// FillNUMANodesWithZero takes a CPUSet containing NUMA node IDs and ensures that each ID is present in MemoryDetails.
// If a NUMA node ID from the CPUSet is not present in the MemoryDetails map, it is added with a value of 0.
// The method returns an updated MemoryDetails map with these changes.
func (d MemoryDetails) FillNUMANodesWithZero(allNUMAs CPUSet) MemoryDetails {
// Clone the original MemoryDetails map
updatedDetails := d.Clone()
// Iterate through all NUMA IDs and ensure they are in the map
for numaID := range allNUMAs.ToSliceInt() {
if _, exists := updatedDetails[numaID]; !exists {
// Add the NUMA ID with a value of 0 if it doesn't exist in the map
updatedDetails[numaID] = 0
}
}
// Return the updated MemoryDetails map
return updatedDetails
}
type MemoryTopology struct {
MemoryDetails MemoryDetails
}
// CPUsPerCore returns the number of logical CPUs
// associated with each core.
func (topo *CPUTopology) CPUsPerCore() int {
if topo.NumCores == 0 {
return 0
}
return topo.NumCPUs / topo.NumCores
}
// CPUsPerSocket returns the number of logical CPUs
// associated with each socket.
func (topo *CPUTopology) CPUsPerSocket() int {
if topo.NumSockets == 0 {
return 0
}
return topo.NumCPUs / topo.NumSockets
}
// CPUsPerNuma returns the number of logical CPUs
// associated with each numa node.
func (topo *CPUTopology) CPUsPerNuma() int {
if topo.NumNUMANodes == 0 {
return 0
}
return topo.NumCPUs / topo.NumNUMANodes
}
// NUMAsPerSocket returns the the number of NUMA
// are associated with each socket.
func (topo *CPUTopology) NUMAsPerSocket() (int, error) {
numasCount := topo.CPUDetails.NUMANodes().Size()
if numasCount%topo.NumSockets != 0 {
return 0, fmt.Errorf("invalid numasCount: %d and socketsCount: %d", numasCount, topo.NumSockets)
}
return numasCount / topo.NumSockets, nil
}
// GetSocketTopology parses the given CPUTopology to a mapping
// from socket id to cpu id lists
func (topo *CPUTopology) GetSocketTopology() map[int]string {
if topo == nil {
return nil
}
socketTopology := make(map[int]string)
for _, socketID := range topo.CPUDetails.Sockets().ToSliceInt() {
socketTopology[socketID] = topo.CPUDetails.NUMANodesInSockets(socketID).String()
}
return socketTopology
}
func GenerateDummyMachineInfo(numaNum int, memoryCapacityGB int) (*info.MachineInfo, error) {
machineInfo := &info.MachineInfo{}
if memoryCapacityGB%numaNum != 0 {
return nil, fmt.Errorf("invalid memoryCapacityGB: %d and NUMA number: %d", memoryCapacityGB, numaNum)
}
perNumaCapacityGB := uint64(memoryCapacityGB / numaNum)
perNumaCapacityQuantity := resource.MustParse(fmt.Sprintf("%dGi", perNumaCapacityGB))
machineInfo.Topology = make([]info.Node, 0, numaNum)
for i := 0; i < numaNum; i++ {
machineInfo.Topology = append(machineInfo.Topology, info.Node{
Id: i,
Memory: uint64(perNumaCapacityQuantity.Value()),
})
}
return machineInfo, nil
}
func GenerateDummyCPUTopology(cpuNum, socketNum, numaNum int) (*CPUTopology, error) {
if numaNum%socketNum != 0 {
return nil, fmt.Errorf("invalid NUMA number: %d and socket number: %d", numaNum, socketNum)
} else if cpuNum%numaNum != 0 {
return nil, fmt.Errorf("invalid cpu number: %d and NUMA number: %d", cpuNum, numaNum)
} else if cpuNum%2 != 0 {
// assume that we should use hyper-threads
return nil, fmt.Errorf("invalid cpu number: %d and NUMA number: %d", cpuNum, numaNum)
}
cpuTopology := new(CPUTopology)
cpuTopology.CPUDetails = make(map[int]CPUInfo)
cpuTopology.NumCPUs = cpuNum
cpuTopology.NumCores = cpuNum / 2
cpuTopology.NumSockets = socketNum
cpuTopology.NumNUMANodes = numaNum
numaPerSocket := numaNum / socketNum
cpusPerNUMA := cpuNum / numaNum
for i := 0; i < socketNum; i++ {
for j := i * numaPerSocket; j < (i+1)*numaPerSocket; j++ {
for k := j * (cpusPerNUMA / 2); k < (j+1)*(cpusPerNUMA/2); k++ {
cpuTopology.CPUDetails[k] = CPUInfo{
NUMANodeID: j,
SocketID: i,
CoreID: k,
}
cpuTopology.CPUDetails[k+cpuNum/2] = CPUInfo{
NUMANodeID: j,
SocketID: i,
CoreID: k,
}
}
}
}
return cpuTopology, nil
}
func GenerateDummyMemoryTopology(numaNum int, memoryCapacity uint64) (*MemoryTopology, error) {
memoryTopology := &MemoryTopology{map[int]uint64{}}
for i := 0; i < numaNum; i++ {
memoryTopology.MemoryDetails[i] = memoryCapacity / uint64(numaNum)
}
return memoryTopology, nil
}
func GenerateDummyExtraTopology(numaNum int) (*ExtraTopologyInfo, error) {
var (
socketNum = 2
distanceNumaInSameSocket = 11
distanceNumaInOtherSocket = 21
)
extraTopology := &ExtraTopologyInfo{
NumaDistanceMap: make(map[int][]NumaDistanceInfo),
SiblingNumaMap: make(map[int]sets.Int),
SiblingNumaAvgMBWAllocatableMap: make(map[int]int64),
SiblingNumaAvgMBWCapacityMap: make(map[int]int64),
}
for i := 0; i < numaNum; i++ {
numaDistanceInfos := make([]NumaDistanceInfo, 0)
for j := 0; j < numaNum; j++ {
if i == j {
continue
} else if i/socketNum == j/socketNum {
numaDistanceInfos = append(numaDistanceInfos, NumaDistanceInfo{
Distance: distanceNumaInSameSocket,
NumaID: j,
})
} else {
numaDistanceInfos = append(numaDistanceInfos, NumaDistanceInfo{
Distance: distanceNumaInOtherSocket,
NumaID: j,
})
}
}
extraTopology.NumaDistanceMap[i] = numaDistanceInfos
extraTopology.SiblingNumaMap[i] = make(sets.Int)
}
return extraTopology, nil
}
// CPUInfo contains the NUMA, socket, and core IDs associated with a CPU.
type CPUInfo struct {
NUMANodeID int
SocketID int
CoreID int
}
// KeepOnly returns a new CPUDetails object with only the supplied cpus.
func (d CPUDetails) KeepOnly(cpus CPUSet) CPUDetails {
result := CPUDetails{}
for cpu, info := range d {
if cpus.Contains(cpu) {
result[cpu] = info
}
}
return result
}
// NUMANodes returns all NUMANode IDs associated with the CPUs in this CPUDetails.
func (d CPUDetails) NUMANodes() CPUSet {
b := NewCPUSet()
for _, info := range d {
b.Add(info.NUMANodeID)
}
return b
}
// NUMANodesInSockets returns all logical NUMANode IDs associated with
// the given socket IDs in this CPUDetails.
func (d CPUDetails) NUMANodesInSockets(ids ...int) CPUSet {
b := NewCPUSet()
for _, id := range ids {
for _, info := range d {
if info.SocketID == id {
b.Add(info.NUMANodeID)
}
}
}
return b
}
// Sockets returns all socket IDs associated with the CPUs in this CPUDetails.
func (d CPUDetails) Sockets() CPUSet {
b := NewCPUSet()
for _, info := range d {
b.Add(info.SocketID)
}
return b
}
// CPUsInSockets returns all logical CPU IDs associated with the given
// socket IDs in this CPUDetails.
func (d CPUDetails) CPUsInSockets(ids ...int) CPUSet {
b := NewCPUSet()
for _, id := range ids {
for cpu, info := range d {
if info.SocketID == id {
b.Add(cpu)
}
}
}
return b
}
// SocketsInNUMANodes returns all logical Socket IDs associated with the
// given NUMANode IDs in this CPUDetails.
func (d CPUDetails) SocketsInNUMANodes(ids ...int) CPUSet {
b := NewCPUSet()
for _, id := range ids {
for _, info := range d {
if info.NUMANodeID == id {
b.Add(info.SocketID)
}
}
}
return b
}
// Cores returns all core IDs associated with the CPUs in this CPUDetails.
func (d CPUDetails) Cores() CPUSet {
b := NewCPUSet()
for _, info := range d {
b.Add(info.CoreID)
}
return b
}
// CoresInNUMANodes returns all core IDs associated with the given
// NUMANode IDs in this CPUDetails.
func (d CPUDetails) CoresInNUMANodes(ids ...int) CPUSet {
b := NewCPUSet()
for _, id := range ids {
for _, info := range d {
if info.NUMANodeID == id {
b.Add(info.CoreID)
}
}
}
return b
}
// CoresInSockets returns all core IDs associated with the given socket
// IDs in this CPUDetails.
func (d CPUDetails) CoresInSockets(ids ...int) CPUSet {
b := NewCPUSet()
for _, id := range ids {
for _, info := range d {
if info.SocketID == id {
b.Add(info.CoreID)
}
}
}
return b
}
// CPUs returns all logical CPU IDs in this CPUDetails.
func (d CPUDetails) CPUs() CPUSet {
b := NewCPUSet()
for cpuID := range d {
b.Add(cpuID)
}
return b
}
// CPUsInNUMANodes returns all logical CPU IDs associated with the given
// NUMANode IDs in this CPUDetails.
func (d CPUDetails) CPUsInNUMANodes(ids ...int) CPUSet {
b := NewCPUSet()
for _, id := range ids {
for cpu, info := range d {
if info.NUMANodeID == id {
b.Add(cpu)
}
}
}
return b
}
// CPUsInCores returns all logical CPU IDs associated with the given
// core IDs in this CPUDetails.
func (d CPUDetails) CPUsInCores(ids ...int) CPUSet {
b := NewCPUSet()
for _, id := range ids {
for cpu, info := range d {
if info.CoreID == id {
b.Add(cpu)
}
}
}
return b
}
// Discover returns CPUTopology based on cadvisor node info
func Discover(machineInfo *info.MachineInfo) (*CPUTopology, *MemoryTopology, error) {
if machineInfo.NumCores == 0 {
return nil, nil, fmt.Errorf("could not detect number of cpus")
}
CPUDetails := CPUDetails{}
numPhysicalCores := 0
memoryTopology := MemoryTopology{MemoryDetails: map[int]uint64{}}
for _, node := range machineInfo.Topology {
memoryTopology.MemoryDetails[node.Id] = node.Memory
numPhysicalCores += len(node.Cores)
for _, core := range node.Cores {
if coreID, err := getUniqueCoreID(core.Threads); err == nil {
for _, cpu := range core.Threads {
CPUDetails[cpu] = CPUInfo{
CoreID: coreID,
SocketID: core.SocketID,
NUMANodeID: node.Id,
}
}
} else {
klog.ErrorS(nil, "Could not get unique coreID for socket",
"socket", core.SocketID, "core", core.Id, "threads", core.Threads)
return nil, nil, err
}
}
}
return &CPUTopology{
NumCPUs: machineInfo.NumCores,
NumSockets: machineInfo.NumSockets,
NumCores: numPhysicalCores,
NumNUMANodes: CPUDetails.NUMANodes().Size(),
CPUDetails: CPUDetails,
}, &memoryTopology, nil
}
// getUniqueCoreID computes coreId as the lowest cpuID
// for a given Threads []int slice. This will assure that coreID's are
// platform unique (opposite to what cAdvisor reports)
func getUniqueCoreID(threads []int) (coreID int, err error) {
if len(threads) == 0 {
return 0, fmt.Errorf("no cpus provided")
}
if len(threads) != NewCPUSet(threads...).Size() {
return 0, fmt.Errorf("cpus provided are not unique")
}
min := threads[0]
for _, thread := range threads[1:] {
if thread < min {
min = thread
}
}
return min, nil
}
// GetNumaAwareAssignments returns a mapping from NUMA id to cpu core
func GetNumaAwareAssignments(topology *CPUTopology, cset CPUSet) (map[int]CPUSet, error) {
if topology == nil {
return nil, fmt.Errorf("GetTopologyAwareAssignmentsByCPUSet got nil cpuset")
}
topologyAwareAssignments := make(map[int]CPUSet)
numaNodes := topology.CPUDetails.NUMANodes()
for _, numaNode := range numaNodes.ToSliceNoSortInt() {
cs := cset.Intersection(topology.CPUDetails.CPUsInNUMANodes(numaNode).Clone())
if cs.Size() > 0 {
topologyAwareAssignments[numaNode] = cs
}
}
return topologyAwareAssignments, nil
}
// CheckNUMACrossSockets judges whether the given NUMA nodes are located
// in different sockets
func CheckNUMACrossSockets(numaNodes []int, cpuTopology *CPUTopology) (bool, error) {
if cpuTopology == nil {
return false, fmt.Errorf("CheckNUMACrossSockets got nil cpuTopology")
}
if len(numaNodes) <= 1 {
return false, nil
}
return cpuTopology.CPUDetails.SocketsInNUMANodes(numaNodes...).Size() > 1, nil
}
type NumaDistanceInfo struct {
NumaID int
Distance int
}
type ExtraTopologyInfo struct {
NumaDistanceMap map[int][]NumaDistanceInfo
SiblingNumaMap map[int]sets.Int
// SiblingNumaAvgMBWAllocatableMap maps NUMA IDs to the allocatable memory bandwidth,
// averaged across each NUMA node and its siblings.
// SiblingNumaAvgMBWCapacityMap maps NUMA IDs to the capacity memory bandwidth,
// averaged similarly.
SiblingNumaAvgMBWAllocatableMap map[int]int64
SiblingNumaAvgMBWCapacityMap map[int]int64
}