/
hardware.py
2413 lines (2011 loc) · 95.5 KB
/
hardware.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright 2014 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import itertools
import re
import typing as ty
import os_resource_classes as orc
import os_traits
from oslo_log import log as logging
from oslo_utils import strutils
from oslo_utils import units
import nova.conf
from nova import exception
from nova.i18n import _
from nova import objects
from nova.objects import fields
from nova.pci import stats
CONF = nova.conf.CONF
LOG = logging.getLogger(__name__)
MEMPAGES_SMALL = -1
MEMPAGES_LARGE = -2
MEMPAGES_ANY = -3
class VTPMConfig(ty.NamedTuple):
version: str
model: str
def get_vcpu_pin_set():
"""Parse ``vcpu_pin_set`` config.
:returns: A set of host CPU IDs that can be used for VCPU and PCPU
allocations.
"""
if not CONF.vcpu_pin_set:
return None
cpuset_ids = parse_cpu_spec(CONF.vcpu_pin_set)
if not cpuset_ids:
msg = _("No CPUs available after parsing 'vcpu_pin_set' config, %r")
raise exception.Invalid(msg % CONF.vcpu_pin_set)
return cpuset_ids
def get_cpu_dedicated_set():
"""Parse ``[compute] cpu_dedicated_set`` config.
:returns: A set of host CPU IDs that can be used for PCPU allocations.
"""
if not CONF.compute.cpu_dedicated_set:
return None
cpu_ids = parse_cpu_spec(CONF.compute.cpu_dedicated_set)
if not cpu_ids:
msg = _("No CPUs available after parsing '[compute] "
"cpu_dedicated_set' config, %r")
raise exception.Invalid(msg % CONF.compute.cpu_dedicated_set)
return cpu_ids
def get_cpu_shared_set():
"""Parse ``[compute] cpu_shared_set`` config.
:returns: A set of host CPU IDs that can be used for emulator threads and,
optionally, for VCPU allocations.
"""
if not CONF.compute.cpu_shared_set:
return None
shared_ids = parse_cpu_spec(CONF.compute.cpu_shared_set)
if not shared_ids:
msg = _("No CPUs available after parsing '[compute] cpu_shared_set' "
"config, %r")
raise exception.Invalid(msg % CONF.compute.cpu_shared_set)
return shared_ids
def parse_cpu_spec(spec: str) -> ty.Set[int]:
"""Parse a CPU set specification.
Each element in the list is either a single CPU number, a range of
CPU numbers, or a caret followed by a CPU number to be excluded
from a previous range.
:param spec: cpu set string eg "1-4,^3,6"
:returns: a set of CPU indexes
"""
cpuset_ids: ty.Set[int] = set()
cpuset_reject_ids: ty.Set[int] = set()
for rule in spec.split(','):
rule = rule.strip()
# Handle multi ','
if len(rule) < 1:
continue
# Note the count limit in the .split() call
range_parts = rule.split('-', 1)
if len(range_parts) > 1:
reject = False
if range_parts[0] and range_parts[0][0] == '^':
reject = True
range_parts[0] = str(range_parts[0][1:])
# So, this was a range; start by converting the parts to ints
try:
start, end = [int(p.strip()) for p in range_parts]
except ValueError:
raise exception.Invalid(_("Invalid range expression %r")
% rule)
# Make sure it's a valid range
if start > end:
raise exception.Invalid(_("Invalid range expression %r")
% rule)
# Add available CPU ids to set
if not reject:
cpuset_ids |= set(range(start, end + 1))
else:
cpuset_reject_ids |= set(range(start, end + 1))
elif rule[0] == '^':
# Not a range, the rule is an exclusion rule; convert to int
try:
cpuset_reject_ids.add(int(rule[1:].strip()))
except ValueError:
raise exception.Invalid(_("Invalid exclusion "
"expression %r") % rule)
else:
# OK, a single CPU to include; convert to int
try:
cpuset_ids.add(int(rule))
except ValueError:
raise exception.Invalid(_("Invalid inclusion "
"expression %r") % rule)
# Use sets to handle the exclusion rules for us
cpuset_ids -= cpuset_reject_ids
return cpuset_ids
def format_cpu_spec(
cpuset: ty.Set[int],
allow_ranges: bool = True,
) -> str:
"""Format a libvirt CPU range specification.
Format a set/list of CPU indexes as a libvirt CPU range
specification. If allow_ranges is true, it will try to detect
continuous ranges of CPUs, otherwise it will just list each CPU
index explicitly.
:param cpuset: set (or list) of CPU indexes
:param allow_ranges: Whether we should attempt to detect continuous ranges
of CPUs.
:returns: a formatted CPU range string
"""
# We attempt to detect ranges, but don't bother with
# trying to do range negations to minimize the overall
# spec string length
if allow_ranges:
ranges: ty.List[ty.List[int]] = []
previndex = None
for cpuindex in sorted(cpuset):
if previndex is None or previndex != (cpuindex - 1):
ranges.append([])
ranges[-1].append(cpuindex)
previndex = cpuindex
parts = []
for entry in ranges:
if len(entry) == 1:
parts.append(str(entry[0]))
else:
parts.append("%d-%d" % (entry[0], entry[len(entry) - 1]))
return ",".join(parts)
else:
return ",".join(str(id) for id in sorted(cpuset))
def get_number_of_serial_ports(flavor, image_meta):
"""Get the number of serial consoles from the flavor or image.
If flavor extra specs is not set, then any image meta value is
permitted. If flavor extra specs *is* set, then this provides the
default serial port count. The image meta is permitted to override
the extra specs, but *only* with a lower value, i.e.:
- flavor hw:serial_port_count=4
VM gets 4 serial ports
- flavor hw:serial_port_count=4 and image hw_serial_port_count=2
VM gets 2 serial ports
- image hw_serial_port_count=6
VM gets 6 serial ports
- flavor hw:serial_port_count=4 and image hw_serial_port_count=6
Abort guest boot - forbidden to exceed flavor value
:param flavor: Flavor object to read extra specs from
:param image_meta: nova.objects.ImageMeta object instance
:raises: exception.ImageSerialPortNumberInvalid if the serial port count
is not a valid integer
:raises: exception.ImageSerialPortNumberExceedFlavorValue if the serial
port count defined in image is greater than that of flavor
:returns: number of serial ports
"""
flavor_num_ports, image_num_ports = _get_flavor_image_meta(
'serial_port_count', flavor, image_meta)
if flavor_num_ports:
try:
flavor_num_ports = int(flavor_num_ports)
except ValueError:
raise exception.ImageSerialPortNumberInvalid(
num_ports=flavor_num_ports)
if flavor_num_ports and image_num_ports:
if image_num_ports > flavor_num_ports:
raise exception.ImageSerialPortNumberExceedFlavorValue()
return image_num_ports
return flavor_num_ports or image_num_ports or 1
class InstanceInfo(object):
def __init__(self, state, internal_id=None):
"""Create a new Instance Info object
:param state: Required. The running state, one of the power_state codes
:param internal_id: Optional. A unique ID for the instance. Need not be
related to the Instance.uuid.
"""
self.state = state
self.internal_id = internal_id
def __eq__(self, other):
return (self.__class__ == other.__class__ and
self.__dict__ == other.__dict__)
def _score_cpu_topology(topology, wanttopology):
"""Compare a topology against a desired configuration.
Calculate a score indicating how well a provided topology matches
against a preferred topology, where:
a score of 3 indicates an exact match for sockets, cores and
threads
a score of 2 indicates a match of sockets and cores, or sockets
and threads, or cores and threads
a score of 1 indicates a match of sockets or cores or threads
a score of 0 indicates no match
:param wanttopology: nova.objects.VirtCPUTopology instance for
preferred topology
:returns: score in range 0 (worst) to 3 (best)
"""
score = 0
if wanttopology.sockets and topology.sockets == wanttopology.sockets:
score = score + 1
if wanttopology.cores and topology.cores == wanttopology.cores:
score = score + 1
if wanttopology.threads and topology.threads == wanttopology.threads:
score = score + 1
return score
def get_cpu_topology_constraints(flavor, image_meta):
"""Get the topology constraints declared in flavor or image
Extracts the topology constraints from the configuration defined in
the flavor extra specs or the image metadata. In the flavor this
will look for:
hw:cpu_sockets - preferred socket count
hw:cpu_cores - preferred core count
hw:cpu_threads - preferred thread count
hw:cpu_max_sockets - maximum socket count
hw:cpu_max_cores - maximum core count
hw:cpu_max_threads - maximum thread count
In the image metadata this will look at:
hw_cpu_sockets - preferred socket count
hw_cpu_cores - preferred core count
hw_cpu_threads - preferred thread count
hw_cpu_max_sockets - maximum socket count
hw_cpu_max_cores - maximum core count
hw_cpu_max_threads - maximum thread count
The image metadata must be strictly lower than any values set in
the flavor. All values are, however, optional.
:param flavor: Flavor object to read extra specs from
:param image_meta: nova.objects.ImageMeta object instance
:raises: exception.ImageVCPULimitsRangeExceeded if the maximum
counts set against the image exceed the maximum counts
set against the flavor
:raises: exception.ImageVCPUTopologyRangeExceeded if the preferred
counts set against the image exceed the maximum counts set
against the image or flavor
:raises: exception.InvalidRequest if one of the provided flavor properties
is a non-integer
:returns: A two-tuple of objects.VirtCPUTopology instances. The
first element corresponds to the preferred topology,
while the latter corresponds to the maximum topology,
based on upper limits.
"""
flavor_max_sockets, image_max_sockets = _get_flavor_image_meta(
'cpu_max_sockets', flavor, image_meta, 0)
flavor_max_cores, image_max_cores = _get_flavor_image_meta(
'cpu_max_cores', flavor, image_meta, 0)
flavor_max_threads, image_max_threads = _get_flavor_image_meta(
'cpu_max_threads', flavor, image_meta, 0)
# image metadata is already of the correct type
try:
flavor_max_sockets = int(flavor_max_sockets)
flavor_max_cores = int(flavor_max_cores)
flavor_max_threads = int(flavor_max_threads)
except ValueError as e:
msg = _('Invalid flavor extra spec. Error: %s') % str(e)
raise exception.InvalidRequest(msg)
LOG.debug("Flavor limits %(sockets)d:%(cores)d:%(threads)d",
{"sockets": flavor_max_sockets,
"cores": flavor_max_cores,
"threads": flavor_max_threads})
LOG.debug("Image limits %(sockets)d:%(cores)d:%(threads)d",
{"sockets": image_max_sockets,
"cores": image_max_cores,
"threads": image_max_threads})
# Image limits are not permitted to exceed the flavor
# limits. ie they can only lower what the flavor defines
if ((flavor_max_sockets and image_max_sockets > flavor_max_sockets) or
(flavor_max_cores and image_max_cores > flavor_max_cores) or
(flavor_max_threads and image_max_threads > flavor_max_threads)):
raise exception.ImageVCPULimitsRangeExceeded(
image_sockets=image_max_sockets,
image_cores=image_max_cores,
image_threads=image_max_threads,
flavor_sockets=flavor_max_sockets,
flavor_cores=flavor_max_cores,
flavor_threads=flavor_max_threads)
max_sockets = image_max_sockets or flavor_max_sockets or 65536
max_cores = image_max_cores or flavor_max_cores or 65536
max_threads = image_max_threads or flavor_max_threads or 65536
flavor_sockets, image_sockets = _get_flavor_image_meta(
'cpu_sockets', flavor, image_meta, 0)
flavor_cores, image_cores = _get_flavor_image_meta(
'cpu_cores', flavor, image_meta, 0)
flavor_threads, image_threads = _get_flavor_image_meta(
'cpu_threads', flavor, image_meta, 0)
try:
flavor_sockets = int(flavor_sockets)
flavor_cores = int(flavor_cores)
flavor_threads = int(flavor_threads)
except ValueError as e:
msg = _('Invalid flavor extra spec. Error: %s') % str(e)
raise exception.InvalidRequest(msg)
LOG.debug("Flavor pref %(sockets)d:%(cores)d:%(threads)d",
{"sockets": flavor_sockets,
"cores": flavor_cores,
"threads": flavor_threads})
LOG.debug("Image pref %(sockets)d:%(cores)d:%(threads)d",
{"sockets": image_sockets,
"cores": image_cores,
"threads": image_threads})
# If the image limits have reduced the flavor limits we might need
# to discard the preferred topology from the flavor
if ((flavor_sockets > max_sockets) or
(flavor_cores > max_cores) or
(flavor_threads > max_threads)):
flavor_sockets = flavor_cores = flavor_threads = 0
# However, image topology is not permitted to exceed image/flavor
# limits
if ((image_sockets > max_sockets) or
(image_cores > max_cores) or
(image_threads > max_threads)):
raise exception.ImageVCPUTopologyRangeExceeded(
image_sockets=image_sockets,
image_cores=image_cores,
image_threads=image_threads,
max_sockets=max_sockets,
max_cores=max_cores,
max_threads=max_threads)
# If no preferred topology was set against the image then use the
# preferred topology from the flavor. We use 'not or' rather than
# 'not and', since if any value is set against the image this
# invalidates the entire set of values from the flavor
if not any((image_sockets, image_cores, image_threads)):
sockets = flavor_sockets
cores = flavor_cores
threads = flavor_threads
else:
sockets = image_sockets
cores = image_cores
threads = image_threads
LOG.debug('Chose sockets=%(sockets)d, cores=%(cores)d, '
'threads=%(threads)d; limits were sockets=%(maxsockets)d, '
'cores=%(maxcores)d, threads=%(maxthreads)d',
{"sockets": sockets, "cores": cores,
"threads": threads, "maxsockets": max_sockets,
"maxcores": max_cores, "maxthreads": max_threads})
return (objects.VirtCPUTopology(sockets=sockets, cores=cores,
threads=threads),
objects.VirtCPUTopology(sockets=max_sockets, cores=max_cores,
threads=max_threads))
def _get_possible_cpu_topologies(vcpus, maxtopology,
allow_threads):
"""Get a list of possible topologies for a vCPU count.
Given a total desired vCPU count and constraints on the maximum
number of sockets, cores and threads, return a list of
objects.VirtCPUTopology instances that represent every possible
topology that satisfies the constraints.
:param vcpus: total number of CPUs for guest instance
:param maxtopology: objects.VirtCPUTopology instance for upper
limits
:param allow_threads: True if the hypervisor supports CPU threads
:raises: exception.ImageVCPULimitsRangeImpossible if it is
impossible to achieve the total vcpu count given
the maximum limits on sockets, cores and threads
:returns: list of objects.VirtCPUTopology instances
"""
# Clamp limits to number of vcpus to prevent
# iterating over insanely large list
maxsockets = min(vcpus, maxtopology.sockets)
maxcores = min(vcpus, maxtopology.cores)
maxthreads = min(vcpus, maxtopology.threads)
if not allow_threads:
maxthreads = 1
LOG.debug("Build topologies for %(vcpus)d vcpu(s) "
"%(maxsockets)d:%(maxcores)d:%(maxthreads)d",
{"vcpus": vcpus, "maxsockets": maxsockets,
"maxcores": maxcores, "maxthreads": maxthreads})
# Figure out all possible topologies that match
# the required vcpus count and satisfy the declared
# limits. If the total vCPU count were very high
# it might be more efficient to factorize the vcpu
# count and then only iterate over its factors, but
# that's overkill right now
possible = []
for s in range(1, maxsockets + 1):
for c in range(1, maxcores + 1):
for t in range(1, maxthreads + 1):
if (t * c * s) != vcpus:
continue
possible.append(
objects.VirtCPUTopology(sockets=s,
cores=c,
threads=t))
# We want to
# - Minimize threads (ie larger sockets * cores is best)
# - Prefer sockets over cores
possible = sorted(possible, reverse=True,
key=lambda x: (x.sockets * x.cores,
x.sockets,
x.threads))
LOG.debug("Got %d possible topologies", len(possible))
if len(possible) == 0:
raise exception.ImageVCPULimitsRangeImpossible(vcpus=vcpus,
sockets=maxsockets,
cores=maxcores,
threads=maxthreads)
return possible
def _sort_possible_cpu_topologies(possible, wanttopology):
"""Sort the topologies in order of preference.
Sort the provided list of possible topologies such that the
configurations which most closely match the preferred topology are
first.
:param possible: list of objects.VirtCPUTopology instances
:param wanttopology: objects.VirtCPUTopology instance for preferred
topology
:returns: sorted list of nova.objects.VirtCPUTopology instances
"""
# Look at possible topologies and score them according
# to how well they match the preferred topologies
# We don't use python's sort(), since we want to
# preserve the sorting done when populating the
# 'possible' list originally
scores: ty.Dict[int, ty.List['objects.VirtCPUTopology']] = (
collections.defaultdict(list)
)
for topology in possible:
score = _score_cpu_topology(topology, wanttopology)
scores[score].append(topology)
# Build list of all possible topologies sorted
# by the match score, best match first
desired = []
desired.extend(scores[3])
desired.extend(scores[2])
desired.extend(scores[1])
desired.extend(scores[0])
return desired
def _get_desirable_cpu_topologies(flavor, image_meta, allow_threads=True):
"""Identify desirable CPU topologies based for given constraints.
Look at the properties set in the flavor extra specs and the image
metadata and build up a list of all possible valid CPU topologies
that can be used in the guest. Then return this list sorted in
order of preference.
:param flavor: objects.Flavor instance to query extra specs from
:param image_meta: nova.objects.ImageMeta object instance
:param allow_threads: if the hypervisor supports CPU threads
:returns: sorted list of objects.VirtCPUTopology instances
"""
LOG.debug("Getting desirable topologies for flavor %(flavor)s "
"and image_meta %(image_meta)s, allow threads: %(threads)s",
{"flavor": flavor, "image_meta": image_meta,
"threads": allow_threads})
preferred, maximum = get_cpu_topology_constraints(flavor, image_meta)
LOG.debug("Topology preferred %(preferred)s, maximum %(maximum)s",
{"preferred": preferred, "maximum": maximum})
possible = _get_possible_cpu_topologies(flavor.vcpus,
maximum,
allow_threads)
LOG.debug("Possible topologies %s", possible)
desired = _sort_possible_cpu_topologies(possible, preferred)
LOG.debug("Sorted desired topologies %s", desired)
return desired
def get_best_cpu_topology(flavor, image_meta, allow_threads=True):
"""Identify best CPU topology for given constraints.
Look at the properties set in the flavor extra specs and the image
metadata and build up a list of all possible valid CPU topologies
that can be used in the guest. Then return the best topology to use
:param flavor: objects.Flavor instance to query extra specs from
:param image_meta: nova.objects.ImageMeta object instance
:param allow_threads: if the hypervisor supports CPU threads
:returns: an objects.VirtCPUTopology instance for best topology
"""
return _get_desirable_cpu_topologies(
flavor, image_meta, allow_threads)[0]
def _numa_cell_supports_pagesize_request(host_cell, inst_cell):
"""Determine whether the cell can accept the request.
:param host_cell: host cell to fit the instance cell onto
:param inst_cell: instance cell we want to fit
:raises: exception.MemoryPageSizeNotSupported if custom page
size not supported in host cell
:returns: the page size able to be handled by host_cell
"""
avail_pagesize = [page.size_kb for page in host_cell.mempages]
avail_pagesize.sort(reverse=True)
def verify_pagesizes(host_cell, inst_cell, avail_pagesize):
inst_cell_mem = inst_cell.memory * units.Ki
for pagesize in avail_pagesize:
if host_cell.can_fit_pagesize(pagesize, inst_cell_mem):
return pagesize
if inst_cell.pagesize == MEMPAGES_SMALL:
return verify_pagesizes(host_cell, inst_cell, avail_pagesize[-1:])
elif inst_cell.pagesize == MEMPAGES_LARGE:
return verify_pagesizes(host_cell, inst_cell, avail_pagesize[:-1])
elif inst_cell.pagesize == MEMPAGES_ANY:
return verify_pagesizes(host_cell, inst_cell, avail_pagesize)
else:
return verify_pagesizes(host_cell, inst_cell, [inst_cell.pagesize])
def _pack_instance_onto_cores(host_cell, instance_cell,
num_cpu_reserved=0):
"""Pack an instance onto a set of siblings.
Calculate the pinning for the given instance and its topology,
making sure that hyperthreads of the instance match up with those
of the host when the pinning takes effect. Also ensure that the
physical cores reserved for hypervisor on this host NUMA node do
not break any thread policies.
Currently the strategy for packing is to prefer siblings and try use
cores evenly by using emptier cores first. This is achieved by the
way we order cores in the sibling_sets structure, and the order in
which we iterate through it.
The main packing loop that iterates over the sibling_sets dictionary
will not currently try to look for a fit that maximizes number of
siblings, but will simply rely on the iteration ordering and picking
the first viable placement.
:param host_cell: objects.NUMACell instance - the host cell that
the instance should be pinned to
:param instance_cell: An instance of objects.InstanceNUMACell
describing the pinning requirements of the
instance
:param num_cpu_reserved: number of pCPUs reserved for hypervisor
:returns: An instance of objects.InstanceNUMACell containing the
pinning information, the physical cores reserved and
potentially a new topology to be exposed to the
instance. None if there is no valid way to satisfy the
sibling requirements for the instance.
"""
# get number of threads per core in host's cell
threads_per_core = max(map(len, host_cell.siblings)) or 1
LOG.debug('Packing an instance onto a set of siblings: '
' host_cell_free_siblings: %(siblings)s'
' instance_cell: %(cells)s'
' host_cell_id: %(host_cell_id)s'
' threads_per_core: %(threads_per_core)s'
' num_cpu_reserved: %(num_cpu_reserved)s',
{'siblings': host_cell.free_siblings,
'cells': instance_cell,
'host_cell_id': host_cell.id,
'threads_per_core': threads_per_core,
'num_cpu_reserved': num_cpu_reserved})
# We build up a data structure that answers the question: 'Given the
# number of threads I want to pack, give me a list of all the available
# sibling sets (or groups thereof) that can accommodate it'
sibling_sets: ty.Dict[int, ty.List[ty.Set[int]]] = (
collections.defaultdict(list)
)
for sib in host_cell.free_siblings:
for threads_no in range(1, len(sib) + 1):
sibling_sets[threads_no].append(sib)
LOG.debug('Built sibling_sets: %(siblings)s', {'siblings': sibling_sets})
pinning = None
threads_no = 1
def _get_pinning(threads_no, sibling_set, instance_cores):
"""Determines pCPUs/vCPUs mapping
Determines the pCPUs/vCPUs mapping regarding the number of
threads which can be used per cores.
:param threads_no: Number of host threads per cores which can
be used to pin vCPUs according to the
policies.
:param sibling_set: List of available threads per host cores
on a specific host NUMA node.
:param instance_cores: Set of vCPUs requested.
NOTE: Depending on how host is configured (HT/non-HT) a thread can
be considered as an entire core.
"""
if threads_no * len(sibling_set) < (len(instance_cores)):
return None
# Determines usable cores according the "threads number"
# constraint.
#
# For a sibling_set=[(0, 1, 2, 3), (4, 5, 6, 7)] and thread_no 1:
# usable_cores=[[0], [4]]
#
# For a sibling_set=[(0, 1, 2, 3), (4, 5, 6, 7)] and thread_no 2:
# usable_cores=[[0, 1], [4, 5]]
usable_cores = list(map(lambda s: list(s)[:threads_no], sibling_set))
# Determines the mapping vCPUs/pCPUs based on the sets of
# usable cores.
#
# For an instance_cores=[2, 3], usable_cores=[[0], [4]]
# vcpus_pinning=[(2, 0), (3, 4)]
vcpus_pinning = list(zip(sorted(instance_cores),
itertools.chain(*usable_cores)))
msg = ("Computed NUMA topology CPU pinning: usable pCPUs: "
"%(usable_cores)s, vCPUs mapping: %(vcpus_pinning)s")
msg_args = {
'usable_cores': usable_cores,
'vcpus_pinning': vcpus_pinning,
}
LOG.info(msg, msg_args)
return vcpus_pinning
def _get_reserved(sibling_set, vcpus_pinning, num_cpu_reserved=0,
cpu_thread_isolate=False):
"""Given available sibling_set, returns the pCPUs reserved
for hypervisor.
:param sibling_set: List of available threads per host cores
on a specific host NUMA node.
:param vcpus_pinning: List of tuple of (pCPU, vCPU) mapping.
:param num_cpu_reserved: Number of additional host CPUs which
need to be reserved.
:param cpu_thread_isolate: True if CPUThreadAllocationPolicy
is ISOLATE.
"""
if not vcpus_pinning:
return None
cpuset_reserved = None
usable_cores = list(map(lambda s: list(s), sibling_set))
if num_cpu_reserved:
# Updates the pCPUs used based on vCPUs pinned to.
# For the case vcpus_pinning=[(0, 0), (1, 2)] and
# usable_cores=[[0, 1], [2, 3], [4, 5]],
# if CPUThreadAllocationPolicy is isolated, we want
# to update usable_cores=[[4, 5]].
# If CPUThreadAllocationPolicy is *not* isolated,
# we want to update usable_cores=[[1],[3],[4, 5]].
for vcpu, pcpu in vcpus_pinning:
for sib in usable_cores:
if pcpu in sib:
if cpu_thread_isolate:
usable_cores.remove(sib)
else:
sib.remove(pcpu)
# Determines the pCPUs reserved for hypervisor
#
# For usable_cores=[[1],[3],[4, 5]], num_cpu_reserved=1
# cpuset_reserved=set([1])
cpuset_reserved = set(list(
itertools.chain(*usable_cores))[:num_cpu_reserved])
msg = ("Computed NUMA topology reserved pCPUs: usable pCPUs: "
"%(usable_cores)s, reserved pCPUs: %(cpuset_reserved)s")
msg_args = {
'usable_cores': usable_cores,
'cpuset_reserved': cpuset_reserved,
}
LOG.info(msg, msg_args)
return cpuset_reserved or None
if (instance_cell.cpu_thread_policy ==
fields.CPUThreadAllocationPolicy.REQUIRE):
LOG.debug("Requested 'require' thread policy for %d cores",
len(instance_cell))
elif (instance_cell.cpu_thread_policy ==
fields.CPUThreadAllocationPolicy.PREFER):
LOG.debug("Requested 'prefer' thread policy for %d cores",
len(instance_cell))
elif (instance_cell.cpu_thread_policy ==
fields.CPUThreadAllocationPolicy.ISOLATE):
LOG.debug("Requested 'isolate' thread policy for %d cores",
len(instance_cell))
else:
LOG.debug("User did not specify a thread policy. Using default "
"for %d cores", len(instance_cell))
if (instance_cell.cpu_thread_policy ==
fields.CPUThreadAllocationPolicy.ISOLATE):
# make sure we have at least one fully free core
if threads_per_core not in sibling_sets:
LOG.debug('Host does not have any fully free thread sibling sets.'
'It is not possible to emulate a non-SMT behavior '
'for the isolate policy without this.')
return
# TODO(stephenfin): Drop this when we drop support for 'vcpu_pin_set'
# NOTE(stephenfin): This is total hack. We're relying on the fact that
# the libvirt driver, which is the only one that currently supports
# pinned CPUs, will set cpuset and pcpuset to the same value if using
# legacy configuration, i.e. 'vcpu_pin_set', as part of
# '_get_host_numa_topology'. They can't be equal otherwise since
# 'cpu_dedicated_set' and 'cpu_shared_set' must be disjoint. Therefore,
# if these are equal, the host that this NUMA cell corresponds to is
# using legacy configuration and it's okay to use the old, "pin a core
# and reserve its siblings" implementation of the 'isolate' policy. If
# they're not, the host is using new-style configuration and we've just
# hit bug #1889633
if threads_per_core != 1 and host_cell.pcpuset != host_cell.cpuset:
LOG.warning(
"Host supports hyperthreads, but instance requested no "
"hyperthreads. This should have been rejected by the "
"scheduler but we likely got here due to the fallback VCPU "
"query. Consider setting '[workarounds] "
"disable_fallback_pcpu_query' to 'True' once hosts are no "
"longer using 'vcpu_pin_set'. Refer to bug #1889633 for more "
"information."
)
return
pinning = _get_pinning(
1, # we only want to "use" one thread per core
sibling_sets[threads_per_core],
instance_cell.pcpuset)
cpuset_reserved = _get_reserved(
sibling_sets[1], pinning, num_cpu_reserved=num_cpu_reserved,
cpu_thread_isolate=True)
if not pinning or (num_cpu_reserved and not cpuset_reserved):
pinning, cpuset_reserved = (None, None)
else: # REQUIRE, PREFER (explicit, implicit)
if (instance_cell.cpu_thread_policy ==
fields.CPUThreadAllocationPolicy.REQUIRE):
# make sure we actually have some siblings to play with
if threads_per_core <= 1:
LOG.info("Host does not support hyperthreading or "
"hyperthreading is disabled, but 'require' "
"threads policy was requested.")
return
# NOTE(ndipanov): We iterate over the sibling sets in descending order
# of cores that can be packed. This is an attempt to evenly distribute
# instances among physical cores
for threads_no, sibling_set in sorted(
(t for t in sibling_sets.items()), reverse=True):
# NOTE(sfinucan): The key difference between the require and
# prefer policies is that require will not settle for non-siblings
# if this is all that is available. Enforce this by ensuring we're
# using sibling sets that contain at least one sibling
if (instance_cell.cpu_thread_policy ==
fields.CPUThreadAllocationPolicy.REQUIRE):
if threads_no <= 1:
LOG.debug('Skipping threads_no: %s, as it does not satisfy'
' the require policy', threads_no)
continue
pinning = _get_pinning(
threads_no, sibling_set,
instance_cell.pcpuset)
cpuset_reserved = _get_reserved(
sibling_sets[1], pinning, num_cpu_reserved=num_cpu_reserved)
if not pinning or (num_cpu_reserved and not cpuset_reserved):
continue
break
# NOTE(sfinucan): If siblings weren't available and we're using PREFER
# (implicitly or explicitly), fall back to linear assignment across
# cores
if (instance_cell.cpu_thread_policy !=
fields.CPUThreadAllocationPolicy.REQUIRE and
not pinning):
# we create a fake sibling set by splitting all sibling sets and
# treating each core as if it has no siblings. This is necessary
# because '_get_pinning' will normally only take the same amount of
# cores ('threads_no' cores) from each sibling set. This is rather
# desirable when we're seeking to apply a thread policy but it is
# less desirable when we only care about resource usage as we do
# here. By treating each core as independent, as we do here, we
# maximize resource usage for almost-full nodes at the expense of a
# possible performance impact to the guest.
sibling_set = [set([x]) for x in itertools.chain(*sibling_sets[1])]
pinning = _get_pinning(
threads_no, sibling_set,
instance_cell.pcpuset)
cpuset_reserved = _get_reserved(
sibling_set, pinning, num_cpu_reserved=num_cpu_reserved)
if not pinning or (num_cpu_reserved and not cpuset_reserved):
return
LOG.debug('Selected cores for pinning: %s, in cell %s', pinning,
host_cell.id)
instance_cell.pin_vcpus(*pinning)
instance_cell.id = host_cell.id
instance_cell.cpuset_reserved = cpuset_reserved
return instance_cell
def _numa_fit_instance_cell(
host_cell: 'objects.NUMACell',
instance_cell: 'objects.InstanceNUMACell',
limits: ty.Optional['objects.NUMATopologyLimit'] = None,
cpuset_reserved: int = 0,
) -> ty.Optional['objects.InstanceNUMACell']:
"""Ensure an instance cell can fit onto a host cell
Ensure an instance cell can fit onto a host cell and, if so, return
a new objects.InstanceNUMACell with the id set to that of the host.
Returns None if the instance cell exceeds the limits of the host.
:param host_cell: host cell to fit the instance cell onto
:param instance_cell: instance cell we want to fit
:param limits: an objects.NUMATopologyLimit or None
:param cpuset_reserved: An int to indicate the number of CPUs overhead
:returns: objects.InstanceNUMACell with the id set to that of the
host, or None
"""
LOG.debug('Attempting to fit instance cell %(cell)s on host_cell '
'%(host_cell)s', {'cell': instance_cell, 'host_cell': host_cell})
if 'pagesize' in instance_cell and instance_cell.pagesize:
# The instance has requested a page size. Verify that the requested
# size is valid and that there are available pages of that size on the
# host.
pagesize = _numa_cell_supports_pagesize_request(
host_cell, instance_cell)
if not pagesize:
LOG.debug('Host does not support requested memory pagesize, '
'or not enough free pages of the requested size. '
'Requested: %d kB', instance_cell.pagesize)
return None
LOG.debug('Selected memory pagesize: %(selected_mem_pagesize)d kB. '
'Requested memory pagesize: %(requested_mem_pagesize)d '
'(small = -1, large = -2, any = -3)',
{'selected_mem_pagesize': pagesize,
'requested_mem_pagesize': instance_cell.pagesize})
instance_cell.pagesize = pagesize
else:
# The instance provides a NUMA topology but does not define any
# particular page size for its memory.
if host_cell.mempages:
# The host supports explicit page sizes. Use a pagesize-aware
# memory check using the smallest available page size.
pagesize = _get_smallest_pagesize(host_cell)
LOG.debug('No specific pagesize requested for instance, '
'selected pagesize: %d', pagesize)
# we want to allow overcommit in this case as we're not using
# hugepages
if not host_cell.can_fit_pagesize(pagesize,
instance_cell.memory * units.Ki,
use_free=False):
LOG.debug('Not enough available memory to schedule instance '
'with pagesize %(pagesize)d. Required: '
'%(required)s, available: %(available)s, total: '
'%(total)s.',
{'required': instance_cell.memory,
'available': host_cell.avail_memory,
'total': host_cell.memory,
'pagesize': pagesize})
return None
else:
# The host does not support explicit page sizes. Ignore pagesizes
# completely.
# NOTE(stephenfin): Do not allow an instance to overcommit against
# itself on any NUMA cell, i.e. with 'ram_allocation_ratio = 2.0'
# on a host with 1GB RAM, we should allow two 1GB instances but not
# one 2GB instance.
if instance_cell.memory > host_cell.memory:
LOG.debug('Not enough host cell memory to fit instance cell. '
'Required: %(required)d, actual: %(actual)d',
{'required': instance_cell.memory,
'actual': host_cell.memory})
return None
# NOTE(stephenfin): As with memory, do not allow an instance to overcommit
# against itself on any NUMA cell
if instance_cell.cpu_policy in (
fields.CPUAllocationPolicy.DEDICATED,
fields.CPUAllocationPolicy.MIXED,
):
required_cpus = len(instance_cell.pcpuset) + cpuset_reserved
if required_cpus > len(host_cell.pcpuset):
LOG.debug('Not enough host cell CPUs to fit instance cell; '
'required: %(required)d + %(cpuset_reserved)d as '
'overhead, actual: %(actual)d', {
'required': len(instance_cell.pcpuset),
'actual': len(host_cell.pcpuset),
'cpuset_reserved': cpuset_reserved
})