-
Notifications
You must be signed in to change notification settings - Fork 168
/
topology-x86.c
1950 lines (1737 loc) · 65.5 KB
/
topology-x86.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Copyright © 2010-2023 Inria. All rights reserved.
* Copyright © 2010-2013 Université Bordeaux
* Copyright © 2010-2011 Cisco Systems, Inc. All rights reserved.
* See COPYING in top-level directory.
*
*
* This backend is only used when the operating system does not export
* the necessary hardware topology information to user-space applications.
* Currently, FreeBSD and NetBSD only add PUs and then fallback to this
* backend for CPU/Cache discovery.
*
* Other backends such as Linux have their own way to retrieve various
* pieces of hardware topology information from the operating system
* on various architectures, without having to use this x86-specific code.
* But this backend is still used after them to annotate some objects with
* additional details (CPU info in Package, Inclusiveness in Caches).
*/
#include "private/autogen/config.h"
#include "hwloc.h"
#include "private/private.h"
#include "private/debug.h"
#include "private/misc.h"
#include "private/cpuid-x86.h"
#include <sys/types.h>
#ifdef HAVE_DIRENT_H
#include <dirent.h>
#endif
#ifdef HAVE_VALGRIND_VALGRIND_H
#include <valgrind/valgrind.h>
#endif
struct hwloc_x86_backend_data_s {
unsigned nbprocs;
hwloc_bitmap_t apicid_set;
int apicid_unique;
char *src_cpuiddump_path;
int is_knl;
int is_hybrid;
int found_die_ids;
int found_complex_ids;
int found_unit_ids;
int found_module_ids;
int found_tile_ids;
};
/************************************
* Management of cpuid dump as input
*/
struct cpuiddump {
unsigned nr;
struct cpuiddump_entry {
unsigned inmask; /* which of ine[abcd]x are set on input */
unsigned ineax;
unsigned inebx;
unsigned inecx;
unsigned inedx;
unsigned outeax;
unsigned outebx;
unsigned outecx;
unsigned outedx;
} *entries;
};
static void
cpuiddump_free(struct cpuiddump *cpuiddump)
{
if (cpuiddump->nr)
free(cpuiddump->entries);
free(cpuiddump);
}
static struct cpuiddump *
cpuiddump_read(const char *dirpath, unsigned idx)
{
struct cpuiddump *cpuiddump;
struct cpuiddump_entry *cur;
size_t filenamelen;
char *filename;
FILE *file;
char line[128];
unsigned nr;
cpuiddump = malloc(sizeof(*cpuiddump));
if (!cpuiddump) {
fprintf(stderr, "Failed to allocate cpuiddump for PU #%u, ignoring cpuiddump.\n", idx);
goto out;
}
filenamelen = strlen(dirpath) + 15;
filename = malloc(filenamelen);
if (!filename)
goto out_with_dump;
snprintf(filename, filenamelen, "%s/pu%u", dirpath, idx);
file = fopen(filename, "r");
if (!file) {
fprintf(stderr, "Could not read dumped cpuid file %s, ignoring cpuiddump.\n", filename);
goto out_with_filename;
}
nr = 0;
while (fgets(line, sizeof(line), file))
nr++;
cpuiddump->entries = malloc(nr * sizeof(struct cpuiddump_entry));
if (!cpuiddump->entries) {
fprintf(stderr, "Failed to allocate %u cpuiddump entries for PU #%u, ignoring cpuiddump.\n", nr, idx);
goto out_with_file;
}
fseek(file, 0, SEEK_SET);
cur = &cpuiddump->entries[0];
nr = 0;
while (fgets(line, sizeof(line), file)) {
if (*line == '#')
continue;
if (sscanf(line, "%x %x %x %x %x => %x %x %x %x",
&cur->inmask,
&cur->ineax, &cur->inebx, &cur->inecx, &cur->inedx,
&cur->outeax, &cur->outebx, &cur->outecx, &cur->outedx) == 9) {
cur++;
nr++;
}
}
cpuiddump->nr = nr;
fclose(file);
free(filename);
return cpuiddump;
out_with_file:
fclose(file);
out_with_filename:
free(filename);
out_with_dump:
free(cpuiddump);
out:
return NULL;
}
static void
cpuiddump_find_by_input(unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx, struct cpuiddump *cpuiddump)
{
unsigned i;
for(i=0; i<cpuiddump->nr; i++) {
struct cpuiddump_entry *entry = &cpuiddump->entries[i];
if ((entry->inmask & 0x1) && *eax != entry->ineax)
continue;
if ((entry->inmask & 0x2) && *ebx != entry->inebx)
continue;
if ((entry->inmask & 0x4) && *ecx != entry->inecx)
continue;
if ((entry->inmask & 0x8) && *edx != entry->inedx)
continue;
*eax = entry->outeax;
*ebx = entry->outebx;
*ecx = entry->outecx;
*edx = entry->outedx;
return;
}
fprintf(stderr, "Couldn't find %x,%x,%x,%x in dumped cpuid, returning 0s.\n",
*eax, *ebx, *ecx, *edx);
*eax = 0;
*ebx = 0;
*ecx = 0;
*edx = 0;
}
static void cpuid_or_from_dump(unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx, struct cpuiddump *src_cpuiddump)
{
if (src_cpuiddump) {
cpuiddump_find_by_input(eax, ebx, ecx, edx, src_cpuiddump);
} else {
hwloc_x86_cpuid(eax, ebx, ecx, edx);
}
}
/*******************************
* Core detection routines and structures
*/
enum hwloc_x86_disc_flags {
HWLOC_X86_DISC_FLAG_FULL = (1<<0), /* discover everything instead of only annotating */
HWLOC_X86_DISC_FLAG_TOPOEXT_NUMANODES = (1<<1) /* use AMD topoext numanode information */
};
#define has_topoext(features) ((features)[6] & (1 << 22))
#define has_x2apic(features) ((features)[4] & (1 << 21))
#define has_hybrid(features) ((features)[18] & (1 << 15))
struct cacheinfo {
hwloc_obj_cache_type_t type;
unsigned level;
unsigned nbthreads_sharing;
unsigned cacheid;
unsigned linesize;
unsigned linepart;
int inclusive;
int ways;
unsigned sets;
unsigned long size;
};
struct procinfo {
unsigned present;
unsigned apicid;
#define PKG 0
#define CORE 1
#define NODE 2
#define UNIT 3
#define TILE 4
#define MODULE 5
#define DIE 6
#define COMPLEX 7
#define HWLOC_X86_PROCINFO_ID_NR 8
unsigned ids[HWLOC_X86_PROCINFO_ID_NR];
unsigned *otherids;
unsigned levels;
unsigned numcaches;
struct cacheinfo *cache;
char cpuvendor[13];
char cpumodel[3*4*4+1];
unsigned cpustepping;
unsigned cpumodelnumber;
unsigned cpufamilynumber;
unsigned hybridcoretype;
unsigned hybridnativemodel;
};
enum cpuid_type {
intel,
amd,
zhaoxin,
hygon,
unknown
};
/* AMD legacy cache information from specific CPUID 0x80000005-6 leaves */
static void setup__amd_cache_legacy(struct procinfo *infos, unsigned level, hwloc_obj_cache_type_t type, unsigned nbthreads_sharing, unsigned cpuid)
{
struct cacheinfo *cache, *tmpcaches;
unsigned cachenum;
unsigned long size = 0;
if (level == 1)
size = ((cpuid >> 24)) << 10;
else if (level == 2)
size = ((cpuid >> 16)) << 10;
else if (level == 3)
size = ((cpuid >> 18)) << 19;
if (!size)
return;
tmpcaches = realloc(infos->cache, (infos->numcaches+1)*sizeof(*infos->cache));
if (!tmpcaches)
/* failed to allocated, ignore that cache */
return;
infos->cache = tmpcaches;
cachenum = infos->numcaches++;
cache = &infos->cache[cachenum];
cache->type = type;
cache->level = level;
cache->nbthreads_sharing = nbthreads_sharing;
cache->linesize = cpuid & 0xff;
cache->linepart = 0;
cache->inclusive = 0; /* old AMD (K8-K10) supposed to have exclusive caches */
if (level == 1) {
cache->ways = (cpuid >> 16) & 0xff;
if (cache->ways == 0xff)
/* Fully associative */
cache->ways = -1;
} else {
static const unsigned ways_tab[] = { 0, 1, 2, 0, 4, 0, 8, 0, 16, 0, 32, 48, 64, 96, 128, -1 };
unsigned ways = (cpuid >> 12) & 0xf;
cache->ways = ways_tab[ways];
}
cache->size = size;
cache->sets = 0;
hwloc_debug("cache L%u t%u linesize %u ways %d size %luKB\n", cache->level, cache->nbthreads_sharing, cache->linesize, cache->ways, cache->size >> 10);
}
/* AMD legacy cache information from CPUID 0x80000005-6 leaves */
static void read_amd_caches_legacy(struct procinfo *infos, struct cpuiddump *src_cpuiddump, unsigned legacy_max_log_proc)
{
unsigned eax, ebx, ecx, edx;
eax = 0x80000005;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
setup__amd_cache_legacy(infos, 1, HWLOC_OBJ_CACHE_DATA, 1, ecx); /* private L1d */
setup__amd_cache_legacy(infos, 1, HWLOC_OBJ_CACHE_INSTRUCTION, 1, edx); /* private L1i */
eax = 0x80000006;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
if (ecx & 0xf000)
/* This is actually supported on Intel but LinePerTag isn't returned in bits 8-11.
* Could be useful if some Intels (at least before Core micro-architecture)
* support this leaf without leaf 0x4.
*/
setup__amd_cache_legacy(infos, 2, HWLOC_OBJ_CACHE_UNIFIED, 1, ecx); /* private L2u */
if (edx & 0xf000)
setup__amd_cache_legacy(infos, 3, HWLOC_OBJ_CACHE_UNIFIED, legacy_max_log_proc, edx); /* package-wide L3u */
}
/* AMD caches from CPUID 0x8000001d leaf (topoext) */
static void read_amd_caches_topoext(struct procinfo *infos, struct cpuiddump *src_cpuiddump)
{
unsigned eax, ebx, ecx, edx;
unsigned cachenum;
struct cacheinfo *cache;
/* the code below doesn't want any other cache yet */
assert(!infos->numcaches);
for (cachenum = 0; cachenum<16 /* guard */; cachenum++) {
eax = 0x8000001d;
ecx = cachenum;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
if ((eax & 0x1f) == 0)
break;
infos->numcaches++;
}
cache = infos->cache = malloc(infos->numcaches * sizeof(*infos->cache));
if (cache) {
for (cachenum = 0; cachenum<16 /* guard */; cachenum++) {
unsigned long linesize, linepart, ways, sets;
eax = 0x8000001d;
ecx = cachenum;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
if ((eax & 0x1f) == 0)
break;
switch (eax & 0x1f) {
case 1: cache->type = HWLOC_OBJ_CACHE_DATA; break;
case 2: cache->type = HWLOC_OBJ_CACHE_INSTRUCTION; break;
default: cache->type = HWLOC_OBJ_CACHE_UNIFIED; break;
}
cache->level = (eax >> 5) & 0x7;
/* Note: actually number of cores */
cache->nbthreads_sharing = ((eax >> 14) & 0xfff) + 1;
cache->linesize = linesize = (ebx & 0xfff) + 1;
cache->linepart = linepart = ((ebx >> 12) & 0x3ff) + 1;
ways = ((ebx >> 22) & 0x3ff) + 1;
if (eax & (1 << 9))
/* Fully associative */
cache->ways = -1;
else
cache->ways = ways;
cache->sets = sets = ecx + 1;
cache->size = linesize * linepart * ways * sets;
cache->inclusive = edx & 0x2;
hwloc_debug("cache %u L%u%c t%u linesize %lu linepart %lu ways %lu sets %lu, size %luKB\n",
cachenum, cache->level,
cache->type == HWLOC_OBJ_CACHE_DATA ? 'd' : cache->type == HWLOC_OBJ_CACHE_INSTRUCTION ? 'i' : 'u',
cache->nbthreads_sharing, linesize, linepart, ways, sets, cache->size >> 10);
cache++;
}
} else {
infos->numcaches = 0;
}
}
/* Intel cache info from CPUID 0x04 leaf */
static void read_intel_caches(struct hwloc_x86_backend_data_s *data, struct procinfo *infos, struct cpuiddump *src_cpuiddump)
{
unsigned level;
struct cacheinfo *tmpcaches;
unsigned eax, ebx, ecx, edx;
unsigned oldnumcaches = infos->numcaches; /* in case we got caches above */
unsigned cachenum;
struct cacheinfo *cache;
for (cachenum = 0; cachenum<16 /* guard */; cachenum++) {
eax = 0x04;
ecx = cachenum;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
hwloc_debug("cache %u type %u\n", cachenum, eax & 0x1f);
if ((eax & 0x1f) == 0)
break;
level = (eax >> 5) & 0x7;
if (data->is_knl && level == 3)
/* KNL reports wrong L3 information (size always 0, cpuset always the entire machine, ignore it */
break;
infos->numcaches++;
}
tmpcaches = realloc(infos->cache, infos->numcaches * sizeof(*infos->cache));
if (!tmpcaches) {
infos->numcaches = oldnumcaches;
} else {
infos->cache = tmpcaches;
cache = &infos->cache[oldnumcaches];
for (cachenum = 0; cachenum<16 /* guard */; cachenum++) {
unsigned long linesize, linepart, ways, sets;
eax = 0x04;
ecx = cachenum;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
if ((eax & 0x1f) == 0)
break;
level = (eax >> 5) & 0x7;
if (data->is_knl && level == 3)
/* KNL reports wrong L3 information (size always 0, cpuset always the entire machine, ignore it */
break;
switch (eax & 0x1f) {
case 1: cache->type = HWLOC_OBJ_CACHE_DATA; break;
case 2: cache->type = HWLOC_OBJ_CACHE_INSTRUCTION; break;
default: cache->type = HWLOC_OBJ_CACHE_UNIFIED; break;
}
cache->level = level;
cache->nbthreads_sharing = ((eax >> 14) & 0xfff) + 1;
cache->linesize = linesize = (ebx & 0xfff) + 1;
cache->linepart = linepart = ((ebx >> 12) & 0x3ff) + 1;
ways = ((ebx >> 22) & 0x3ff) + 1;
if (eax & (1 << 9))
/* Fully associative */
cache->ways = -1;
else
cache->ways = ways;
cache->sets = sets = ecx + 1;
cache->size = linesize * linepart * ways * sets;
cache->inclusive = edx & 0x2;
hwloc_debug("cache %u L%u%c t%u linesize %lu linepart %lu ways %lu sets %lu, size %luKB\n",
cachenum, cache->level,
cache->type == HWLOC_OBJ_CACHE_DATA ? 'd' : cache->type == HWLOC_OBJ_CACHE_INSTRUCTION ? 'i' : 'u',
cache->nbthreads_sharing, linesize, linepart, ways, sets, cache->size >> 10);
cache++;
}
}
}
/* AMD core/thread info from CPUID 0x80000008 leaf */
static void read_amd_cores_legacy(struct procinfo *infos, struct cpuiddump *src_cpuiddump)
{
unsigned eax, ebx, ecx, edx;
unsigned max_nbcores;
unsigned max_nbthreads;
unsigned coreidsize;
unsigned logprocid;
unsigned threadid __hwloc_attribute_unused;
eax = 0x80000008;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
coreidsize = (ecx >> 12) & 0xf;
hwloc_debug("core ID size: %u\n", coreidsize);
if (!coreidsize) {
max_nbcores = (ecx & 0xff) + 1;
} else
max_nbcores = 1 << coreidsize;
hwloc_debug("Thus max # of cores: %u\n", max_nbcores);
/* No multithreaded AMD for this old CPUID leaf */
max_nbthreads = 1 ;
hwloc_debug("and max # of threads: %u\n", max_nbthreads);
/* legacy_max_log_proc is deprecated, it can be smaller than max_nbcores,
* which is the maximum number of cores that the processor could theoretically support
* (see "Multiple Core Calculation" in the AMD CPUID specification).
* Recompute packageid/coreid accordingly.
*/
infos->ids[PKG] = infos->apicid / max_nbcores;
logprocid = infos->apicid % max_nbcores;
infos->ids[CORE] = logprocid / max_nbthreads;
threadid = logprocid % max_nbthreads;
hwloc_debug("this is thread %u of core %u\n", threadid, infos->ids[CORE]);
}
/* AMD unit/node from CPUID 0x8000001e leaf (topoext) */
static void read_amd_cores_topoext(struct hwloc_x86_backend_data_s *data, struct procinfo *infos, unsigned long flags, struct cpuiddump *src_cpuiddump)
{
unsigned apic_id, nodes_per_proc = 0;
unsigned eax, ebx, ecx, edx;
eax = 0x8000001e;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
infos->apicid = apic_id = eax;
if (flags & HWLOC_X86_DISC_FLAG_TOPOEXT_NUMANODES) {
if (infos->cpufamilynumber == 0x16) {
/* ecx is reserved */
infos->ids[NODE] = 0;
nodes_per_proc = 1;
} else {
/* AMD other families or Hygon family 18h */
infos->ids[NODE] = ecx & 0xff;
nodes_per_proc = ((ecx >> 8) & 7) + 1;
}
if ((infos->cpufamilynumber == 0x15 && nodes_per_proc > 2)
|| ((infos->cpufamilynumber == 0x17 || infos->cpufamilynumber == 0x18) && nodes_per_proc > 4)
|| (infos->cpufamilynumber == 0x19 && nodes_per_proc > 1)) {
hwloc_debug("warning: undefined nodes_per_proc value %u, assuming it means %u\n", nodes_per_proc, nodes_per_proc);
}
}
if (infos->cpufamilynumber <= 0x16) { /* topoext appeared in 0x15 and compute-units were only used in 0x15 and 0x16 */
unsigned cores_per_unit;
/* coreid was obtained from read_amd_cores_legacy() earlier */
infos->ids[UNIT] = ebx & 0xff;
data->found_unit_ids = 1;
cores_per_unit = ((ebx >> 8) & 0xff) + 1;
hwloc_debug("topoext %08x, %u nodes, node %u, %u cores in unit %u\n", apic_id, nodes_per_proc, infos->ids[NODE], cores_per_unit, infos->ids[UNIT]);
/* coreid and unitid are package-wide (core 0-15 and unit 0-7 on 16-core 2-NUMAnode processor).
* The Linux kernel reduces theses to NUMA-node-wide (by applying %core_per_node and %unit_per node respectively).
* It's not clear if we should do this as well.
*/
} else {
unsigned threads_per_core;
infos->ids[CORE] = ebx & 0xff;
threads_per_core = ((ebx >> 8) & 0xff) + 1;
hwloc_debug("topoext %08x, %u nodes, node %u, %u threads in core %u\n", apic_id, nodes_per_proc, infos->ids[NODE], threads_per_core, infos->ids[CORE]);
}
}
/* Intel core/thread or even die/module/tile from CPUID 0x0b or 0x1f leaves (v1 and v2 extended topology enumeration)
* or AMD complex/ccd from CPUID 0x80000026 (extended CPU topology)
*/
static void read_extended_topo(struct hwloc_x86_backend_data_s *data, struct procinfo *infos, unsigned leaf, enum cpuid_type cpuid_type, struct cpuiddump *src_cpuiddump)
{
unsigned level, apic_nextshift, apic_type, apic_id = 0, apic_shift = 0, id;
unsigned threadid __hwloc_attribute_unused = 0; /* shut-up compiler */
unsigned eax, ebx, ecx = 0, edx;
int apic_packageshift = 0;
for (level = 0; level<32 /* guard */; level++) {
ecx = level;
eax = leaf;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
/* Intel specifies that the 0x0b/0x1f loop should stop when we get "invalid domain" (0 in ecx[8:15])
* (if so, we also get 0 in eax/ebx for invalid subleaves).
* However AMD rather says that the 0x80000026/0x0b loop should stop when we get "no thread at this level" (0 in ebx[0:15]).
* Zhaoxin follows the Intel specs but also returns "no thread at this level" for the last *valid* level (at least on KH-4000).
* From the Linux kernel code, it's very likely that AMD also returns "invalid domain"
* (because detect_extended_topology() uses that for all x86 CPUs)
* but keep with the official doc until AMD can clarify that (see #593).
*/
if (cpuid_type == amd) {
if (!(ebx & 0xffff))
break;
} else {
if (!(ecx & 0xff00))
break;
}
apic_packageshift = eax & 0x1f;
}
if (level) {
infos->otherids = malloc(level * sizeof(*infos->otherids));
if (infos->otherids) {
infos->levels = level;
for (level = 0; level<32 /* guard */; level++) {
ecx = level;
eax = leaf;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
if (cpuid_type == amd) {
if (!(ebx & 0xffff))
break;
} else {
if (!(ecx & 0xff00))
break;
}
apic_nextshift = eax & 0x1f;
apic_type = (ecx & 0xff00) >> 8;
apic_id = edx;
id = (apic_id >> apic_shift) & ((1 << (apic_packageshift - apic_shift)) - 1);
hwloc_debug("x2APIC %08x %u: nextshift %u nextnumber %2u type %u id %2u\n",
apic_id,
level,
apic_nextshift,
ebx & 0xffff /* number of threads in next level */,
apic_type,
id);
infos->apicid = apic_id;
infos->otherids[level] = UINT_MAX;
switch (apic_type) {
case 1:
threadid = id;
break;
case 2:
infos->ids[CORE] = id;
break;
case 3:
if (leaf == 0x80000026) {
data->found_complex_ids = 1;
infos->ids[COMPLEX] = id;
} else {
data->found_module_ids = 1;
infos->ids[MODULE] = id;
}
break;
case 4:
if (leaf == 0x80000026) {
data->found_die_ids = 1;
infos->ids[DIE] = id;
} else {
data->found_tile_ids = 1;
infos->ids[TILE] = id;
}
break;
case 5:
if (leaf == 0x80000026) {
goto unknown_type;
} else {
data->found_die_ids = 1;
infos->ids[DIE] = id;
}
break;
case 6:
/* TODO: "DieGrp" on Intel */
/* fallthrough */
default:
unknown_type:
hwloc_debug("x2APIC %u: unknown type %u\n", level, apic_type);
infos->otherids[level] = apic_id >> apic_shift;
break;
}
apic_shift = apic_nextshift;
}
infos->apicid = apic_id;
infos->ids[PKG] = apic_id >> apic_shift;
hwloc_debug("x2APIC remainder: %u\n", infos->ids[PKG]);
hwloc_debug("this is thread %u of core %u\n", threadid, infos->ids[CORE]);
}
}
}
/* Fetch information from the processor itself thanks to cpuid and store it in
* infos for summarize to analyze them globally */
static void look_proc(struct hwloc_backend *backend, struct procinfo *infos, unsigned long flags, unsigned highest_cpuid, unsigned highest_ext_cpuid, unsigned *features, enum cpuid_type cpuid_type, struct cpuiddump *src_cpuiddump)
{
struct hwloc_x86_backend_data_s *data = backend->private_data;
unsigned eax, ebx, ecx = 0, edx;
unsigned cachenum;
struct cacheinfo *cache;
unsigned regs[4];
unsigned legacy_max_log_proc; /* not valid on Intel processors with > 256 threads, or when cpuid 0x80000008 is supported */
unsigned legacy_log_proc_id;
unsigned _model, _extendedmodel, _family, _extendedfamily;
infos->present = 1;
/* Get apicid, legacy_max_log_proc, packageid, legacy_log_proc_id from cpuid 0x01 */
eax = 0x01;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
infos->apicid = ebx >> 24;
if (edx & (1 << 28)) {
legacy_max_log_proc = 1 << hwloc_flsl(((ebx >> 16) & 0xff) - 1);
} else {
hwloc_debug("HTT bit not set in CPUID 0x01.edx, assuming legacy_max_log_proc = 1\n");
legacy_max_log_proc = 1;
}
hwloc_debug("APIC ID 0x%02x legacy_max_log_proc %u\n", infos->apicid, legacy_max_log_proc);
infos->ids[PKG] = infos->apicid / legacy_max_log_proc;
legacy_log_proc_id = infos->apicid % legacy_max_log_proc;
hwloc_debug("phys %u legacy thread %u\n", infos->ids[PKG], legacy_log_proc_id);
/* Get cpu model/family/stepping numbers from same cpuid */
_model = (eax>>4) & 0xf;
_extendedmodel = (eax>>16) & 0xf;
_family = (eax>>8) & 0xf;
_extendedfamily = (eax>>20) & 0xff;
if ((cpuid_type == intel || cpuid_type == amd || cpuid_type == hygon) && _family == 0xf) {
infos->cpufamilynumber = _family + _extendedfamily;
} else {
infos->cpufamilynumber = _family;
}
if ((cpuid_type == intel && (_family == 0x6 || _family == 0xf))
|| ((cpuid_type == amd || cpuid_type == hygon) && _family == 0xf)
|| (cpuid_type == zhaoxin && (_family == 0x6 || _family == 0x7))) {
infos->cpumodelnumber = _model + (_extendedmodel << 4);
} else {
infos->cpumodelnumber = _model;
}
infos->cpustepping = eax & 0xf;
if (cpuid_type == intel && infos->cpufamilynumber == 0x6 &&
(infos->cpumodelnumber == 0x57 || infos->cpumodelnumber == 0x85))
data->is_knl = 1; /* KNM is the same as KNL */
/* Get cpu vendor string from cpuid 0x00 */
memset(regs, 0, sizeof(regs));
regs[0] = 0;
cpuid_or_from_dump(®s[0], ®s[1], ®s[3], ®s[2], src_cpuiddump);
memcpy(infos->cpuvendor, regs+1, 4*3);
/* infos was calloc'ed, already ends with \0 */
/* Get cpu model string from cpuid 0x80000002-4 */
if (highest_ext_cpuid >= 0x80000004) {
memset(regs, 0, sizeof(regs));
regs[0] = 0x80000002;
cpuid_or_from_dump(®s[0], ®s[1], ®s[2], ®s[3], src_cpuiddump);
memcpy(infos->cpumodel, regs, 4*4);
regs[0] = 0x80000003;
cpuid_or_from_dump(®s[0], ®s[1], ®s[2], ®s[3], src_cpuiddump);
memcpy(infos->cpumodel + 4*4, regs, 4*4);
regs[0] = 0x80000004;
cpuid_or_from_dump(®s[0], ®s[1], ®s[2], ®s[3], src_cpuiddump);
memcpy(infos->cpumodel + 4*4*2, regs, 4*4);
/* infos was calloc'ed, already ends with \0 */
}
if ((cpuid_type != amd && cpuid_type != hygon) && highest_cpuid >= 0x04) {
/* Get core/thread information from first cache reported by cpuid 0x04
* (not supported on AMD)
*/
eax = 0x04;
ecx = 0;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
if ((eax & 0x1f) != 0) {
/* cache looks valid */
unsigned max_nbcores;
unsigned max_nbthreads;
unsigned threadid __hwloc_attribute_unused;
hwloc_debug("Trying to get core/thread IDs from 0x04...\n");
max_nbcores = ((eax >> 26) & 0x3f) + 1;
hwloc_debug("found %u cores max\n", max_nbcores);
/* some VMs (e.g. issue#525) don't report valid information, check things before dividing by 0. */
if (!max_nbcores) {
hwloc_debug("cannot detect core/thread IDs from 0x04 without a valid max of cores\n");
} else {
max_nbthreads = legacy_max_log_proc / max_nbcores;
hwloc_debug("found %u threads max\n", max_nbthreads);
if (!max_nbthreads) {
hwloc_debug("cannot detect core/thread IDs from 0x04 without a valid max of threads\n");
} else {
threadid = legacy_log_proc_id % max_nbthreads;
infos->ids[CORE] = legacy_log_proc_id / max_nbthreads;
hwloc_debug("this is thread %u of core %u\n", threadid, infos->ids[CORE]);
}
}
}
}
if (highest_cpuid >= 0x1a && has_hybrid(features)) {
/* Get hybrid cpu information from cpuid 0x1a on Intel */
eax = 0x1a;
ecx = 0;
cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump);
infos->hybridcoretype = eax >> 24;
infos->hybridnativemodel = eax & 0xffffff;
data->is_hybrid = 1;
}
/*********************************************************************************
* Get the hierarchy of thread, core, die, package, etc. from CPU-specific leaves
*/
if (cpuid_type != intel && cpuid_type != zhaoxin && highest_ext_cpuid >= 0x80000008 && !has_x2apic(features)) {
/* Get core/thread information from cpuid 0x80000008
* (not supported on Intel)
* We could ignore this codepath when x2apic is supported, but we may need
* nodeids if HWLOC_X86_TOPOEXT_NUMANODES is set.
*/
read_amd_cores_legacy(infos, src_cpuiddump);
}
if (cpuid_type != intel && cpuid_type != zhaoxin && has_topoext(features)) {
/* Get apicid, nodeid, unitid/coreid from cpuid 0x8000001e (AMD topology extension).
* Requires read_amd_cores_legacy() for coreid on family 0x15-16.
*
* Only needed when x2apic supported if NUMA nodes are needed.
*/
read_amd_cores_topoext(data, infos, flags, src_cpuiddump);
}
if ((cpuid_type == amd) && highest_ext_cpuid >= 0x80000026) {
/* Get socket/die/complex/core/thread information from cpuid 0x80000026
* (AMD Extended CPU Topology)
*/
read_extended_topo(data, infos, 0x80000026, cpuid_type, src_cpuiddump);
} else if ((cpuid_type == intel || cpuid_type == zhaoxin) && highest_cpuid >= 0x1f) {
/* Get package/die/module/tile/core/thread information from cpuid 0x1f
* (Intel v2 Extended Topology Enumeration)
*/
read_extended_topo(data, infos, 0x1f, cpuid_type, src_cpuiddump);
} else if ((cpuid_type == intel || cpuid_type == amd || cpuid_type == zhaoxin)
&& highest_cpuid >= 0x0b && has_x2apic(features)) {
/* Get package/core/thread information from cpuid 0x0b
* (Intel v1 Extended Topology Enumeration)
*/
read_extended_topo(data, infos, 0x0b, cpuid_type, src_cpuiddump);
}
/**************************************
* Get caches from CPU-specific leaves
*/
infos->numcaches = 0;
infos->cache = NULL;
if (cpuid_type != intel && cpuid_type != zhaoxin && has_topoext(features)) {
/* Get cache information from cpuid 0x8000001d (AMD topology extension) */
read_amd_caches_topoext(infos, src_cpuiddump);
} else if (cpuid_type != intel && cpuid_type != zhaoxin && highest_ext_cpuid >= 0x80000006) {
/* If there's no topoext,
* get cache information from cpuid 0x80000005 and 0x80000006.
* (not supported on Intel)
* It looks like we cannot have 0x80000005 without 0x80000006.
*/
read_amd_caches_legacy(infos, src_cpuiddump, legacy_max_log_proc);
}
if ((cpuid_type != amd && cpuid_type != hygon) && highest_cpuid >= 0x04) {
/* Get cache information from cpuid 0x04
* (not supported on AMD)
*/
read_intel_caches(data, infos, src_cpuiddump);
}
/* Now that we have all info, compute cacheids and apply quirks */
for (cachenum = 0; cachenum < infos->numcaches; cachenum++) {
cache = &infos->cache[cachenum];
/* default cacheid value */
cache->cacheid = infos->apicid / cache->nbthreads_sharing;
if (cpuid_type == intel) {
/* round nbthreads_sharing to nearest power of two to build a mask (for clearing lower bits) */
unsigned bits = hwloc_flsl(cache->nbthreads_sharing-1);
unsigned mask = ~((1U<<bits) - 1);
cache->cacheid = infos->apicid & mask;
} else if (cpuid_type == amd) {
/* AMD quirks */
if (infos->cpufamilynumber >= 0x17 && cache->level == 3) {
/* AMD family 0x19 always shares L3 between 16 APIC ids (8 HT cores).
* while Family 0x17 shares between 8 APIC ids (4 HT cores).
* But many models have less APIC ids enabled and reported in nbthreads_sharing.
* It means we must round-up nbthreads_sharing to the nearest power of 2
* before computing cacheid.
*/
unsigned nbapics_sharing = cache->nbthreads_sharing;
if (nbapics_sharing & (nbapics_sharing-1))
/* not a power of two, round-up */
nbapics_sharing = 1U<<(1+hwloc_ffsl(nbapics_sharing));
cache->cacheid = infos->apicid / nbapics_sharing;
} else if (infos->cpufamilynumber== 0x10 && infos->cpumodelnumber == 0x9
&& cache->level == 3
&& (cache->ways == -1 || (cache->ways % 2 == 0)) && cache->nbthreads_sharing >= 8) {
/* Fix AMD family 0x10 model 0x9 (Magny-Cours) with 8 or 12 cores.
* The L3 (and its associativity) is actually split into two halves).
*/
if (cache->nbthreads_sharing == 16)
cache->nbthreads_sharing = 12; /* nbthreads_sharing is a power of 2 but the processor actually has 8 or 12 cores */
cache->nbthreads_sharing /= 2;
cache->size /= 2;
if (cache->ways != -1)
cache->ways /= 2;
/* AMD Magny-Cours 12-cores processor reserve APIC ids as AAAAAABBBBBB....
* among first L3 (A), second L3 (B), and unexisting cores (.).
* On multi-socket servers, L3 in non-first sockets may have APIC id ranges
* such as [16-21] that are not aligned on multiple of nbthreads_sharing (6).
* That means, we can't just compare apicid/nbthreads_sharing to identify siblings.
*/
cache->cacheid = (infos->apicid % legacy_max_log_proc) / cache->nbthreads_sharing /* cacheid within the package */
+ 2 * (infos->apicid / legacy_max_log_proc); /* add 2 caches per previous package */
} else if (infos->cpufamilynumber == 0x15
&& (infos->cpumodelnumber == 0x1 /* Bulldozer */ || infos->cpumodelnumber == 0x2 /* Piledriver */)
&& cache->level == 3 && cache->nbthreads_sharing == 6) {
/* AMD Bulldozer and Piledriver 12-core processors have same APIC ids as Magny-Cours above,
* but we can't merge the checks because the original nbthreads_sharing must be exactly 6 here.
*/
cache->cacheid = (infos->apicid % legacy_max_log_proc) / cache->nbthreads_sharing /* cacheid within the package */
+ 2 * (infos->apicid / legacy_max_log_proc); /* add 2 cache per previous package */
}
} else if (cpuid_type == hygon) {
if (infos->cpufamilynumber == 0x18
&& cache->level == 3 && cache->nbthreads_sharing == 6) {
/* Hygon family 0x18 always shares L3 between 8 APIC ids,
* even when only 6 APIC ids are enabled and reported in nbthreads_sharing
* (on 24-core CPUs).
*/
cache->cacheid = infos->apicid / 8;
}
}
}
if (hwloc_bitmap_isset(data->apicid_set, infos->apicid))
data->apicid_unique = 0;
else
hwloc_bitmap_set(data->apicid_set, infos->apicid);
}
static void
hwloc_x86_add_cpuinfos(hwloc_obj_t obj, struct procinfo *info, int replace)
{
char number[12];
if (info->cpuvendor[0])
hwloc__add_info_nodup(&obj->infos, &obj->infos_count, "CPUVendor", info->cpuvendor, replace);
snprintf(number, sizeof(number), "%u", info->cpufamilynumber);
hwloc__add_info_nodup(&obj->infos, &obj->infos_count, "CPUFamilyNumber", number, replace);
snprintf(number, sizeof(number), "%u", info->cpumodelnumber);
hwloc__add_info_nodup(&obj->infos, &obj->infos_count, "CPUModelNumber", number, replace);
if (info->cpumodel[0]) {
const char *c = info->cpumodel;
while (*c == ' ')
c++;
hwloc__add_info_nodup(&obj->infos, &obj->infos_count, "CPUModel", c, replace);
}
snprintf(number, sizeof(number), "%u", info->cpustepping);
hwloc__add_info_nodup(&obj->infos, &obj->infos_count, "CPUStepping", number, replace);
}
static void
hwloc_x86_add_groups(hwloc_topology_t topology,
struct procinfo *infos,
unsigned nbprocs,
hwloc_bitmap_t remaining_cpuset,
unsigned type,
const char *subtype,
unsigned kind,
int dont_merge)
{
hwloc_bitmap_t obj_cpuset;
hwloc_obj_t obj;
unsigned i, j;
while ((i = hwloc_bitmap_first(remaining_cpuset)) != (unsigned) -1) {
unsigned packageid = infos[i].ids[PKG];
unsigned id = infos[i].ids[type];
if (id == (unsigned)-1) {
hwloc_bitmap_clr(remaining_cpuset, i);
continue;
}
obj_cpuset = hwloc_bitmap_alloc();
for (j = i; j < nbprocs; j++) {
if (infos[j].ids[type] == (unsigned) -1) {
hwloc_bitmap_clr(remaining_cpuset, j);
continue;
}
if (infos[j].ids[PKG] == packageid && infos[j].ids[type] == id) {
hwloc_bitmap_set(obj_cpuset, j);
hwloc_bitmap_clr(remaining_cpuset, j);
}
}
obj = hwloc_alloc_setup_object(topology, HWLOC_OBJ_GROUP, id);
obj->cpuset = obj_cpuset;
obj->subtype = strdup(subtype);
obj->attr->group.kind = kind;
obj->attr->group.dont_merge = dont_merge;
hwloc_debug_2args_bitmap("os %s %u has cpuset %s\n",
subtype, id, obj_cpuset);
hwloc__insert_object_by_cpuset(topology, NULL, obj, "x86:group");
}
}
/* Analyse information stored in infos, and build/annotate topology levels accordingly */
static void summarize(struct hwloc_backend *backend, struct procinfo *infos, unsigned long flags)
{
struct hwloc_topology *topology = backend->topology;
struct hwloc_x86_backend_data_s *data = backend->private_data;
unsigned nbprocs = data->nbprocs;
hwloc_bitmap_t complete_cpuset = hwloc_bitmap_alloc();
unsigned i, j, l, level;
int one = -1;
hwloc_bitmap_t remaining_cpuset;
int gotnuma = 0;
int fulldiscovery = (flags & HWLOC_X86_DISC_FLAG_FULL);
#ifdef HWLOC_DEBUG
hwloc_debug("\nSummary of x86 CPUID topology:\n");
for(i=0; i<nbprocs; i++) {
hwloc_debug("PU %u present=%u apicid=%u on PKG %d CORE %d DIE %d NODE %d\n",
i, infos[i].present, infos[i].apicid,
infos[i].ids[PKG], infos[i].ids[CORE], infos[i].ids[DIE], infos[i].ids[NODE]);
}
hwloc_debug("\n");
#endif