-
Notifications
You must be signed in to change notification settings - Fork 519
/
summaryallocation.go
1794 lines (1510 loc) · 55.1 KB
/
summaryallocation.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package opencost
import (
"errors"
"fmt"
"strings"
"sync"
"time"
"github.com/opencost/opencost/core/pkg/filter/ast"
"github.com/opencost/opencost/core/pkg/filter/matcher"
"github.com/opencost/opencost/core/pkg/log"
"github.com/opencost/opencost/core/pkg/util/timeutil"
)
// SummaryAllocation summarizes an Allocation, keeping only fields necessary
// for providing a high-level view of identifying the Allocation over a period
// of time (Start, End) over which it ran, and inspecting the associated per-
// resource costs (subtotaled with adjustments), total cost, and efficiency.
//
// SummaryAllocation does not have a concept of Window (i.e. the time period
// within which it is defined, as opposed to the Start and End times). That
// context must be provided by a SummaryAllocationSet.
type SummaryAllocation struct {
Name string `json:"name"`
Properties *AllocationProperties `json:"-"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
CPUCoreRequestAverage float64 `json:"cpuCoreRequestAverage"`
CPUCoreUsageAverage float64 `json:"cpuCoreUsageAverage"`
CPUCost float64 `json:"cpuCost"`
GPUCost float64 `json:"gpuCost"`
NetworkCost float64 `json:"networkCost"`
LoadBalancerCost float64 `json:"loadBalancerCost"`
PVCost float64 `json:"pvCost"`
RAMBytesRequestAverage float64 `json:"ramByteRequestAverage"`
RAMBytesUsageAverage float64 `json:"ramByteUsageAverage"`
RAMCost float64 `json:"ramCost"`
SharedCost float64 `json:"sharedCost"`
ExternalCost float64 `json:"externalCost"`
Share bool `json:"-"`
UnmountedPVCost float64 `json:"-"`
}
// NewSummaryAllocation converts an Allocation to a SummaryAllocation by
// dropping unnecessary fields and consolidating others (e.g. adjustments).
// Reconciliation happens here because that process is synonymous with the
// consolidation of adjustment fields.
func NewSummaryAllocation(alloc *Allocation, reconcile, reconcileNetwork bool) *SummaryAllocation {
if alloc == nil {
return nil
}
sa := &SummaryAllocation{
Name: alloc.Name,
Properties: alloc.Properties,
Start: alloc.Start,
End: alloc.End,
CPUCoreRequestAverage: alloc.CPUCoreRequestAverage,
CPUCoreUsageAverage: alloc.CPUCoreUsageAverage,
CPUCost: alloc.CPUCost + alloc.CPUCostAdjustment,
GPUCost: alloc.GPUCost + alloc.GPUCostAdjustment,
NetworkCost: alloc.NetworkCost + alloc.NetworkCostAdjustment,
LoadBalancerCost: alloc.LoadBalancerCost + alloc.LoadBalancerCostAdjustment,
PVCost: alloc.PVCost() + alloc.PVCostAdjustment,
RAMBytesRequestAverage: alloc.RAMBytesRequestAverage,
RAMBytesUsageAverage: alloc.RAMBytesUsageAverage,
RAMCost: alloc.RAMCost + alloc.RAMCostAdjustment,
SharedCost: alloc.SharedCost,
ExternalCost: alloc.ExternalCost,
UnmountedPVCost: alloc.UnmountedPVCost,
}
// Revert adjustments if reconciliation is off. If only network
// reconciliation is off, only revert network adjustment.
if !reconcile {
sa.CPUCost -= alloc.CPUCostAdjustment
sa.GPUCost -= alloc.GPUCostAdjustment
sa.NetworkCost -= alloc.NetworkCostAdjustment
sa.LoadBalancerCost -= alloc.LoadBalancerCostAdjustment
sa.PVCost -= alloc.PVCostAdjustment
sa.RAMCost -= alloc.RAMCostAdjustment
} else if !reconcileNetwork {
sa.NetworkCost -= alloc.NetworkCostAdjustment
}
// If the allocation is unmounted, set UnmountedPVCost to the full PVCost.
if sa.IsUnmounted() {
sa.UnmountedPVCost = sa.PVCost
}
return sa
}
// Add sums two SummaryAllocations, adding the given SummaryAllocation to the
// receiving one, thus mutating the receiver. For performance reasons, it
// simply drops Properties, so a SummaryAllocation can only be Added once.
func (sa *SummaryAllocation) Add(that *SummaryAllocation) error {
if sa == nil || that == nil {
return errors.New("cannot Add a nil SummaryAllocation")
}
// Once Added, a SummaryAllocation has no Properties. This saves us from
// having to compute the intersection of two sets of Properties, which is
// expensive.
sa.Properties = nil
// Sum non-cumulative fields by turning them into cumulative, adding them,
// and then converting them back into averages after minutes have been
// combined (just below).
cpuReqCoreMins := sa.CPUCoreRequestAverage * sa.Minutes()
cpuReqCoreMins += that.CPUCoreRequestAverage * that.Minutes()
cpuUseCoreMins := sa.CPUCoreUsageAverage * sa.Minutes()
cpuUseCoreMins += that.CPUCoreUsageAverage * that.Minutes()
ramReqByteMins := sa.RAMBytesRequestAverage * sa.Minutes()
ramReqByteMins += that.RAMBytesRequestAverage * that.Minutes()
ramUseByteMins := sa.RAMBytesUsageAverage * sa.Minutes()
ramUseByteMins += that.RAMBytesUsageAverage * that.Minutes()
// Expand Start and End to be the "max" of among the given Allocations
if that.Start.Before(sa.Start) {
sa.Start = that.Start
}
if that.End.After(sa.End) {
sa.End = that.End
}
// Convert cumulative request and usage back into rates
if sa.Minutes() > 0 {
sa.CPUCoreRequestAverage = cpuReqCoreMins / sa.Minutes()
sa.CPUCoreUsageAverage = cpuUseCoreMins / sa.Minutes()
sa.RAMBytesRequestAverage = ramReqByteMins / sa.Minutes()
sa.RAMBytesUsageAverage = ramUseByteMins / sa.Minutes()
} else {
sa.CPUCoreRequestAverage = 0.0
sa.CPUCoreUsageAverage = 0.0
sa.RAMBytesRequestAverage = 0.0
sa.RAMBytesUsageAverage = 0.0
}
// Sum all cumulative cost fields
sa.CPUCost += that.CPUCost
sa.ExternalCost += that.ExternalCost
sa.GPUCost += that.GPUCost
sa.LoadBalancerCost += that.LoadBalancerCost
sa.NetworkCost += that.NetworkCost
sa.PVCost += that.PVCost
sa.RAMCost += that.RAMCost
sa.SharedCost += that.SharedCost
return nil
}
// Clone copies the SummaryAllocation and returns the copy
func (sa *SummaryAllocation) Clone() *SummaryAllocation {
return &SummaryAllocation{
Name: sa.Name,
Properties: sa.Properties.Clone(),
Start: sa.Start,
End: sa.End,
CPUCoreRequestAverage: sa.CPUCoreRequestAverage,
CPUCoreUsageAverage: sa.CPUCoreUsageAverage,
CPUCost: sa.CPUCost,
GPUCost: sa.GPUCost,
NetworkCost: sa.NetworkCost,
LoadBalancerCost: sa.LoadBalancerCost,
PVCost: sa.PVCost,
RAMBytesRequestAverage: sa.RAMBytesRequestAverage,
RAMBytesUsageAverage: sa.RAMBytesUsageAverage,
RAMCost: sa.RAMCost,
SharedCost: sa.SharedCost,
ExternalCost: sa.ExternalCost,
}
}
// CPUEfficiency is the ratio of usage to request. If there is no request and
// no usage or cost, then efficiency is zero. If there is no request, but there
// is usage or cost, then efficiency is 100%.
func (sa *SummaryAllocation) CPUEfficiency() float64 {
if sa == nil || sa.IsIdle() {
return 0.0
}
if sa.CPUCoreRequestAverage > 0 {
return sa.CPUCoreUsageAverage / sa.CPUCoreRequestAverage
}
if sa.CPUCoreUsageAverage == 0.0 || sa.CPUCost == 0.0 {
return 0.0
}
return 1.0
}
func (sa *SummaryAllocation) Equal(that *SummaryAllocation) bool {
if sa == nil || that == nil {
return false
}
if sa.Name != that.Name {
return false
}
if sa.Start != that.Start {
return false
}
if sa.End != that.End {
return false
}
if sa.CPUCoreRequestAverage != that.CPUCoreRequestAverage {
return false
}
if sa.CPUCoreUsageAverage != that.CPUCoreUsageAverage {
return false
}
if sa.CPUCost != that.CPUCost {
return false
}
if sa.GPUCost != that.GPUCost {
return false
}
if sa.NetworkCost != that.NetworkCost {
return false
}
if sa.LoadBalancerCost != that.LoadBalancerCost {
return false
}
if sa.PVCost != that.PVCost {
return false
}
if sa.RAMBytesRequestAverage != that.RAMBytesRequestAverage {
return false
}
if sa.RAMBytesUsageAverage != that.RAMBytesUsageAverage {
return false
}
if sa.RAMCost != that.RAMCost {
return false
}
if sa.SharedCost != that.SharedCost {
return false
}
if sa.ExternalCost != that.ExternalCost {
return false
}
return true
}
func (sa *SummaryAllocation) generateKey(aggregateBy []string, labelConfig *LabelConfig) string {
if sa == nil {
return ""
}
return sa.Properties.GenerateKey(aggregateBy, labelConfig)
}
// IsExternal is true if the given SummaryAllocation represents external costs.
func (sa *SummaryAllocation) IsExternal() bool {
if sa == nil {
return false
}
return strings.Contains(sa.Name, ExternalSuffix)
}
// IsIdle is true if the given SummaryAllocation represents idle costs.
func (sa *SummaryAllocation) IsIdle() bool {
if sa == nil {
return false
}
return strings.Contains(sa.Name, IdleSuffix)
}
// IsUnallocated is true if the given SummaryAllocation represents unallocated
// costs.
func (sa *SummaryAllocation) IsUnallocated() bool {
if sa == nil {
return false
}
return strings.Contains(sa.Name, UnallocatedSuffix)
}
// IsUnmounted is true if the given SummaryAllocation represents unmounted
// volume costs.
func (sa *SummaryAllocation) IsUnmounted() bool {
if sa == nil {
return false
}
return strings.Contains(sa.Name, UnmountedSuffix)
}
// Minutes returns the number of minutes the SummaryAllocation represents, as
// defined by the difference between the end and start times.
func (sa *SummaryAllocation) Minutes() float64 {
if sa == nil {
return 0.0
}
return sa.End.Sub(sa.Start).Minutes()
}
// RAMEfficiency is the ratio of usage to request. If there is no request and
// no usage or cost, then efficiency is zero. If there is no request, but there
// is usage or cost, then efficiency is 100%.
func (sa *SummaryAllocation) RAMEfficiency() float64 {
if sa == nil || sa.IsIdle() {
return 0.0
}
if sa.RAMBytesRequestAverage > 0 {
return sa.RAMBytesUsageAverage / sa.RAMBytesRequestAverage
}
if sa.RAMBytesUsageAverage == 0.0 || sa.RAMCost == 0.0 {
return 0.0
}
return 1.0
}
// TotalCost is the total cost of the SummaryAllocation
func (sa *SummaryAllocation) TotalCost() float64 {
if sa == nil {
return 0.0
}
return sa.CPUCost + sa.GPUCost + sa.RAMCost + sa.PVCost + sa.NetworkCost + sa.LoadBalancerCost + sa.SharedCost + sa.ExternalCost
}
// TotalEfficiency is the cost-weighted average of CPU and RAM efficiency. If
// there is no cost at all, then efficiency is zero.
func (sa *SummaryAllocation) TotalEfficiency() float64 {
if sa == nil || sa.IsIdle() {
return 0.0
}
if sa.RAMCost+sa.CPUCost > 0 {
ramCostEff := sa.RAMEfficiency() * sa.RAMCost
cpuCostEff := sa.CPUEfficiency() * sa.CPUCost
return (ramCostEff + cpuCostEff) / (sa.CPUCost + sa.RAMCost)
}
return 0.0
}
// SummaryAllocationSet stores a set of SummaryAllocations, each with a unique
// name, that share a window. An AllocationSet is mutable, so treat it like a
// threadsafe map.
type SummaryAllocationSet struct {
sync.RWMutex
externalKeys map[string]bool
idleKeys map[string]bool
SummaryAllocations map[string]*SummaryAllocation `json:"allocations"`
Window Window `json:"window"`
}
// NewSummaryAllocationSet converts an AllocationSet to a SummaryAllocationSet.
// Filter functions, keep functions, and reconciliation parameters are
// required for unfortunate reasons to do with performance and legacy order-of-
// operations details, as well as the fact that reconciliation has been
// pushed down to the conversion step between Allocation and SummaryAllocation.
//
// This filter is an AllocationMatcher, not an AST, because at this point we
// already have the data and want to make sure that the filter has already
// gone through a compile step to deal with things like aliases.
func NewSummaryAllocationSet(as *AllocationSet, filter, keep AllocationMatcher, reconcile, reconcileNetwork bool) *SummaryAllocationSet {
if as == nil {
return nil
}
// If we can know the exact size of the map, use it. If filters or sharing
// functions are present, we can't know the size, so we make a default map.
var sasMap map[string]*SummaryAllocation
if filter == nil {
// No filters, so make the map of summary allocations exactly the size
// of the origin allocation set.
sasMap = make(map[string]*SummaryAllocation, len(as.Allocations))
} else {
// There are filters, so start with a standard map
sasMap = make(map[string]*SummaryAllocation)
}
sas := &SummaryAllocationSet{
SummaryAllocations: sasMap,
Window: as.Window.Clone(),
}
for _, alloc := range as.Allocations {
// First, detect if the allocation should be kept. If so, mark it as
// such, insert it, and continue.
if keep != nil && keep.Matches(alloc) {
sa := NewSummaryAllocation(alloc, reconcile, reconcileNetwork)
sa.Share = true
sas.Insert(sa)
continue
}
// If the allocation does not pass any of the given filter functions,
// do not insert it into the set.
if filter != nil && !filter.Matches(alloc) {
continue
}
err := sas.Insert(NewSummaryAllocation(alloc, reconcile, reconcileNetwork))
if err != nil {
log.Errorf("SummaryAllocation: error inserting summary of %s", alloc.Name)
}
}
for key := range as.ExternalKeys {
sas.externalKeys[key] = true
}
for key := range as.IdleKeys {
sas.idleKeys[key] = true
}
return sas
}
// Clone creates a deep copy of the SummaryAllocationSet
func (sas *SummaryAllocationSet) Clone() *SummaryAllocationSet {
sas.RLock()
defer sas.RUnlock()
externalKeys := make(map[string]bool, len(sas.externalKeys))
for k, v := range sas.externalKeys {
externalKeys[k] = v
}
idleKeys := make(map[string]bool, len(sas.idleKeys))
for k, v := range sas.idleKeys {
idleKeys[k] = v
}
summaryAllocations := make(map[string]*SummaryAllocation, len(sas.SummaryAllocations))
for k, v := range sas.SummaryAllocations {
summaryAllocations[k] = v.Clone()
}
return &SummaryAllocationSet{
externalKeys: externalKeys,
idleKeys: idleKeys,
SummaryAllocations: summaryAllocations,
Window: sas.Window.Clone(),
}
}
// Add sums two SummaryAllocationSets, which Adds all SummaryAllocations in the
// given SummaryAllocationSet to their counterparts in the receiving set. Add
// also expands the Window to include both constituent Windows, in the case
// that Add is being used from accumulating (as opposed to aggregating). For
// performance reasons, the function may return either a new set, or an
// unmodified original, so it should not be assumed that the original sets are
// safeuly usable after calling Add.
func (sas *SummaryAllocationSet) Add(that *SummaryAllocationSet) (*SummaryAllocationSet, error) {
if sas == nil || len(sas.SummaryAllocations) == 0 {
return that, nil
}
if that == nil || len(that.SummaryAllocations) == 0 {
return sas, nil
}
if sas.Window.IsOpen() {
return nil, errors.New("cannot add a SummaryAllocationSet with an open window")
}
// Set start, end to min(start), max(end)
start := *sas.Window.Start()
end := *sas.Window.End()
if that.Window.Start().Before(start) {
start = *that.Window.Start()
}
if that.Window.End().After(end) {
end = *that.Window.End()
}
acc := &SummaryAllocationSet{
SummaryAllocations: make(map[string]*SummaryAllocation, len(sas.SummaryAllocations)),
Window: NewClosedWindow(start, end),
}
sas.RLock()
defer sas.RUnlock()
that.RLock()
defer that.RUnlock()
for _, alloc := range sas.SummaryAllocations {
err := acc.Insert(alloc)
if err != nil {
return nil, err
}
}
for _, alloc := range that.SummaryAllocations {
err := acc.Insert(alloc)
if err != nil {
return nil, err
}
}
return acc, nil
}
func (sas *SummaryAllocationSet) GetUnmountedPVCost() float64 {
upvc := 0.0
for _, sa := range sas.SummaryAllocations {
upvc += sa.UnmountedPVCost
}
return upvc
}
// AggregateBy aggregates the Allocations in the given AllocationSet by the given
// AllocationProperty. This will only be legal if the AllocationSet is divisible by the
// given AllocationProperty; e.g. Containers can be divided by Namespace, but not vice-a-versa.
func (sas *SummaryAllocationSet) AggregateBy(aggregateBy []string, options *AllocationAggregationOptions) error {
if sas == nil || len(sas.SummaryAllocations) == 0 {
return nil
}
if sas.Window.IsOpen() {
return errors.New("cannot aggregate a SummaryAllocationSet with an open window")
}
if options == nil {
options = &AllocationAggregationOptions{}
}
if options.LabelConfig == nil {
options.LabelConfig = NewLabelConfig()
}
var filter AllocationMatcher
if options.Filter == nil {
filter = &matcher.AllPass[*Allocation]{}
} else {
compiler := NewAllocationMatchCompiler(options.LabelConfig)
var err error
filter, err = compiler.Compile(options.Filter)
if err != nil {
return fmt.Errorf("compiling filter '%s': %w", ast.ToPreOrderShortString(options.Filter), err)
}
}
if filter == nil {
return fmt.Errorf("unexpected nil filter")
}
// Check if we have any work to do; if not, then early return. If
// aggregateBy is nil, we don't aggregate anything. On the other hand,
// an empty slice implies that we should aggregate everything. (See
// generateKey for why that makes sense.)
shouldAggregate := aggregateBy != nil
shouldKeep := len(options.SharedHourlyCosts) > 0 || options.Share != nil
if !shouldAggregate && !shouldKeep {
return nil
}
// The order of operations for aggregating a SummaryAllotionSet is as
// follows:
//
// 1. Partition external, idle, and shared allocations into separate sets.
// Also, create the resultSet into which the results will be aggregated.
//
// 2. Record resource totals for shared costs and unmounted volumes so
// that we can account for them in computing idle coefficients.
//
// 3. Retrieve pre-computed allocation resource totals, which will be used
// to compute idle sharing coefficients.
//
// 4. Convert shared hourly cost into a cumulative allocation to share,
// and insert it into the share set.
//
// 5. Compute sharing coefficients per-aggregation, if sharing resources.
//
// 6. Distribute idle allocations according to the idle coefficients.
//
// 7. Record allocation resource totals (after filtration) if filters have
// been applied. (Used for filtering proportional amount of idle.)
//
// 8. Generate aggregation key and insert allocation into the output set
//
// 9. If idle is shared and resources are shared, it's probable that some
// amount of idle cost will be shared with a shared resource.
// Distribute that idle cost, if it exists, among the respective shared
// allocations before sharing them with the aggregated allocations.
//
// 10. Apply idle filtration, which "filters" the idle cost, or scales it
// by the proportion of allocation resources remaining after filters
// have been applied.
//
// 11. Distribute shared resources according to sharing coefficients.
//
// 12. Insert external allocations into the result set.
//
// 13. Insert any undistributed idle, in the case that idle
// coefficients end up being zero and some idle is not shared.
//
// 14. Combine all idle allocations into a single idle allocation, unless
// the option to keep idle split by cluster or node is enabled.
// 1. Partition external, idle, and shared allocations into separate sets.
// Also, create the resultSet into which the results will be aggregated.
// resultSet will collect the aggregated allocations
resultSet := &SummaryAllocationSet{
Window: sas.Window.Clone(),
}
// externalSet will collect external allocations
externalSet := &SummaryAllocationSet{
Window: sas.Window.Clone(),
}
// idleSet will be shared among resultSet after initial aggregation
// is complete
idleSet := &SummaryAllocationSet{
Window: sas.Window.Clone(),
}
// shareSet will be shared among resultSet after initial aggregation
// is complete
shareSet := &SummaryAllocationSet{
Window: sas.Window.Clone(),
}
sas.Lock()
defer sas.Unlock()
// 2. Record resource totals for shared costs, aggregating by cluster or by
// node (depending on if idle is partitioned by cluster or node) so that we
// can account for them in computing idle coefficients. Do the same for
// unmounted volume costs, which only require a total cost.
sharedResourceTotals := map[string]*AllocationTotals{}
totalUnmountedCost := 0.0
// 1 & 2. Identify set membership and aggregate aforementioned totals.
for _, sa := range sas.SummaryAllocations {
if sa.Share {
var key string
if options.IdleByNode {
key = fmt.Sprintf("%s/%s", sa.Properties.Cluster, sa.Properties.Node)
} else {
key = sa.Properties.Cluster
}
if _, ok := sharedResourceTotals[key]; !ok {
sharedResourceTotals[key] = &AllocationTotals{}
}
sharedResourceTotals[key].CPUCost += sa.CPUCost
sharedResourceTotals[key].GPUCost += sa.GPUCost
sharedResourceTotals[key].LoadBalancerCost += sa.LoadBalancerCost
sharedResourceTotals[key].NetworkCost += sa.NetworkCost
sharedResourceTotals[key].PersistentVolumeCost += sa.PVCost
sharedResourceTotals[key].RAMCost += sa.RAMCost
sharedResourceTotals[key].UnmountedPVCost += sa.UnmountedPVCost
shareSet.Insert(sa)
delete(sas.SummaryAllocations, sa.Name)
continue
}
// External allocations get aggregated post-hoc (see step 6) and do
// not necessarily contain complete sets of properties, so they are
// moved to a separate AllocationSet.
if sa.IsExternal() {
delete(sas.externalKeys, sa.Name)
delete(sas.SummaryAllocations, sa.Name)
externalSet.Insert(sa)
continue
}
// Idle allocations should be separated into idleSet if they are to be
// shared later on. If they are not to be shared, then add them to the
// resultSet like any other allocation.
if sa.IsIdle() {
delete(sas.idleKeys, sa.Name)
delete(sas.SummaryAllocations, sa.Name)
if options.ShareIdle == ShareEven || options.ShareIdle == ShareWeighted {
idleSet.Insert(sa)
} else {
resultSet.Insert(sa)
}
continue
}
// Track total unmounted cost because it must be taken out of total
// allocated costs for sharing coefficients.
if sa.IsUnmounted() {
totalUnmountedCost += sa.TotalCost()
}
}
// It's possible that no more un-shared, non-idle, non-external allocations
// remain at this point. This always results in an emptySet, so return early.
if len(sas.SummaryAllocations) == 0 {
sas.SummaryAllocations = map[string]*SummaryAllocation{}
return nil
}
// 3. Retrieve pre-computed allocation resource totals, which will be used
// to compute idle coefficients, based on the ratio of an allocation's per-
// resource cost to the per-resource totals of that allocation's cluster or
// node. Whether to perform this operation based on cluster or node is an
// option. (See IdleByNode documentation; defaults to idle-by-cluster.)
var allocTotals map[string]*AllocationTotals
var ok bool
if options.AllocationTotalsStore != nil {
if options.IdleByNode {
allocTotals, ok = options.AllocationTotalsStore.GetAllocationTotalsByNode(*sas.Window.Start(), *sas.Window.End())
if !ok {
return fmt.Errorf("nil allocation resource totals by node for %s", sas.Window)
}
} else {
allocTotals, ok = options.AllocationTotalsStore.GetAllocationTotalsByCluster(*sas.Window.Start(), *sas.Window.End())
if !ok {
return fmt.Errorf("nil allocation resource totals by cluster for %s", sas.Window)
}
}
}
// If reconciliation has been fully or partially disabled, clear the
// relevant adjustments from the alloc totals
if allocTotals != nil && (!options.Reconcile || !options.ReconcileNetwork) {
if !options.Reconcile {
for _, tot := range allocTotals {
tot.ClearAdjustments()
}
} else if !options.ReconcileNetwork {
for _, tot := range allocTotals {
tot.NetworkCostAdjustment = 0.0
}
}
}
// If filters have been applied, then we need to record allocation resource
// totals after filtration (i.e. the allocations that are present) so that
// we can identify the proportion of idle cost to keep. That is, we should
// only return the idle cost that would be shared with the remaining
// allocations, even if we're keeping idle separate. The totals should be
// recorded by idle-key (cluster or node, depending on the IdleByNode
// option). Instantiating this map is a signal to record the totals.
var allocTotalsAfterFilters map[string]*AllocationTotals
if len(resultSet.idleKeys) > 0 && options.Filter != nil {
allocTotalsAfterFilters = make(map[string]*AllocationTotals, len(resultSet.idleKeys))
}
// If we're recording allocTotalsAfterFilters and there are shared costs,
// then record those resource totals here so that idle for those shared
// resources gets included.
if allocTotalsAfterFilters != nil {
for key, rt := range sharedResourceTotals {
if _, ok := allocTotalsAfterFilters[key]; !ok {
allocTotalsAfterFilters[key] = &AllocationTotals{}
}
// Record only those fields required for computing idle
allocTotalsAfterFilters[key].CPUCost += rt.CPUCost
allocTotalsAfterFilters[key].GPUCost += rt.GPUCost
allocTotalsAfterFilters[key].RAMCost += rt.RAMCost
}
}
// 4. Convert shared hourly cost into a cumulative allocation to share,
// and insert it into the share set.
for name, cost := range options.SharedHourlyCosts {
if cost > 0.0 {
hours := sas.Window.Hours()
// If set ends in the future, adjust hours accordingly
diff := time.Since(*sas.Window.End())
if diff < 0.0 {
hours += diff.Hours()
}
totalSharedCost := cost * hours
shareSet.Insert(&SummaryAllocation{
Name: fmt.Sprintf("%s/%s", name, SharedSuffix),
Properties: &AllocationProperties{},
Start: *sas.Window.Start(),
End: *sas.Window.End(),
SharedCost: totalSharedCost,
})
}
}
// Sharing coefficients are recorded by post-aggregation-key (e.g. if
// aggregating by namespace, then the key will be the namespace) and only
// need to be recorded if there are shared resources. Instantiating this
// map is the signal to record sharing coefficients.
var sharingCoeffs map[string]float64
if len(shareSet.SummaryAllocations) > 0 {
sharingCoeffs = map[string]float64{}
}
// Loop over all remaining SummaryAllocations (after filters, sharing, &c.)
// doing the following, in this order:
// 5. Compute sharing coefficients, if there are shared resources
// 6. Distribute idle cost, if sharing idle
// 7. Record allocTotalsAfterFiltration, if filters have been applied
// 8. Aggregate by key
for _, sa := range sas.SummaryAllocations {
// Generate key to use for aggregation-by-key and allocation name
key := sa.generateKey(aggregateBy, options.LabelConfig)
// 5. Incrementally add to sharing coefficients before adding idle
// cost, which would skew the coefficients. These coefficients will be
// later divided by a total, turning them into a coefficient between
// 0.0 and 1.0.
// NOTE: SummaryAllocation does not support ShareEven, so only record
// by cost for cost-weighted distribution.
// if sharingCoeffs != nil {
if sharingCoeffs != nil && !sa.IsUnmounted() {
sharingCoeffs[key] += sa.TotalCost() - sa.SharedCost - sa.UnmountedPVCost
}
// 6. Distribute idle allocations according to the idle coefficients.
// NOTE: if idle allocation is off (i.e. options.ShareIdle: ShareNone)
// then all idle allocations will be in the resultSet at this point, so
// idleSet will be empty and we won't enter this block.
if len(idleSet.SummaryAllocations) > 0 {
for _, idle := range idleSet.SummaryAllocations {
// Idle key is either cluster or node, as determined by the
// IdleByNode option.
var key string
// Only share idle allocation with current allocation (sa) if
// the relevant properties match (i.e. cluster and/or node)
if idle.Properties.Cluster != sa.Properties.Cluster {
continue
}
key = idle.Properties.Cluster
if options.IdleByNode {
if idle.Properties.Node != sa.Properties.Node {
continue
}
key = fmt.Sprintf("%s/%s", idle.Properties.Cluster, idle.Properties.Node)
}
cpuCoeff, gpuCoeff, ramCoeff := ComputeIdleCoefficients(options.ShareIdle, key, sa.CPUCost, sa.GPUCost, sa.RAMCost, allocTotals)
sa.CPUCost += idle.CPUCost * cpuCoeff
sa.GPUCost += idle.GPUCost * gpuCoeff
sa.RAMCost += idle.RAMCost * ramCoeff
}
}
// The key becomes the allocation's name, which is used as the key by
// which the allocation is inserted into the set.
sa.Name = key
// If merging unallocated allocations, rename all unallocated
// allocations as simply __unallocated__
if options.MergeUnallocated && sa.IsUnallocated() {
sa.Name = UnallocatedSuffix
}
// 7. Record filtered resource totals for idle allocation filtration,
// only if necessary.
if allocTotalsAfterFilters != nil {
key := sa.Properties.Cluster
if options.IdleByNode {
key = fmt.Sprintf("%s/%s", sa.Properties.Cluster, sa.Properties.Node)
}
if _, ok := allocTotalsAfterFilters[key]; !ok {
allocTotalsAfterFilters[key] = &AllocationTotals{}
}
allocTotalsAfterFilters[key].CPUCost += sa.CPUCost
allocTotalsAfterFilters[key].GPUCost += sa.GPUCost
allocTotalsAfterFilters[key].RAMCost += sa.RAMCost
}
// 8. Inserting the allocation with the generated key for a name
// performs the actual aggregation step.
resultSet.Insert(sa)
}
// 9. If idle is shared and resources are shared, it's probable that some
// amount of idle cost will be shared with a shared resource. Distribute
// that idle cost, if it exists, among the respective shared allocations
// before sharing them with the aggregated allocations.
if len(idleSet.SummaryAllocations) > 0 && len(shareSet.SummaryAllocations) > 0 {
for _, sa := range shareSet.SummaryAllocations {
for _, idle := range idleSet.SummaryAllocations {
var key string
// Only share idle allocation with current allocation (sa) if
// the relevant property matches (i.e. Cluster or Node,
// depending on which idle sharing option is selected)
if options.IdleByNode {
if idle.Properties.Cluster != sa.Properties.Cluster || idle.Properties.Node != sa.Properties.Node {
continue
}
key = fmt.Sprintf("%s/%s", idle.Properties.Cluster, idle.Properties.Node)
} else {
if idle.Properties.Cluster != sa.Properties.Cluster {
continue
}
key = idle.Properties.Cluster
}
cpuCoeff, gpuCoeff, ramCoeff := ComputeIdleCoefficients(options.ShareIdle, key, sa.CPUCost, sa.GPUCost, sa.RAMCost, allocTotals)
sa.CPUCost += idle.CPUCost * cpuCoeff
sa.GPUCost += idle.GPUCost * gpuCoeff
sa.RAMCost += idle.RAMCost * ramCoeff
}
}
}
// 10. Apply idle filtration, which "filters" the idle cost, i.e. scales
// idle allocation costs per-resource by the proportion of allocation
// resources remaining after filtering. In effect, this returns only the
// idle costs that would have been shared with the remaining allocations,
// even if idle is kept separated.
if allocTotalsAfterFilters != nil {
for idleKey := range resultSet.idleKeys {
ia := resultSet.SummaryAllocations[idleKey]
var key string
if options.IdleByNode {
key = fmt.Sprintf("%s/%s", ia.Properties.Cluster, ia.Properties.Node)
} else {
key = ia.Properties.Cluster
}
// Percentage of idle that should remain after filters are applied,
// which equals the proportion of filtered-to-actual cost.
cpuFilterCoeff := 0.0
if allocTotals[key].TotalCPUCost() > 0.0 {
filteredAlloc, ok := allocTotalsAfterFilters[key]
if ok {
cpuFilterCoeff = filteredAlloc.TotalCPUCost() / allocTotals[key].TotalCPUCost()
} else {
cpuFilterCoeff = 0.0
}
}
gpuFilterCoeff := 0.0
if allocTotals[key].TotalGPUCost() > 0.0 {
filteredAlloc, ok := allocTotalsAfterFilters[key]
if ok {
gpuFilterCoeff = filteredAlloc.TotalGPUCost() / allocTotals[key].TotalGPUCost()
} else {
gpuFilterCoeff = 0.0
}
}
ramFilterCoeff := 0.0
if allocTotals[key].TotalRAMCost() > 0.0 {
filteredAlloc, ok := allocTotalsAfterFilters[key]
if ok {
ramFilterCoeff = filteredAlloc.TotalRAMCost() / allocTotals[key].TotalRAMCost()
} else {
ramFilterCoeff = 0.0
}
}
ia.CPUCost *= cpuFilterCoeff
ia.GPUCost *= gpuFilterCoeff
ia.RAMCost *= ramFilterCoeff
}
}
// 11. Distribute shared resources according to sharing coefficients.
// NOTE: ShareEven is not supported
if len(shareSet.SummaryAllocations) > 0 {
shareCoeffSum := 0.0
sharingCoeffDenominator := 0.0