/
kmp.h
3874 lines (3342 loc) · 147 KB
/
kmp.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*! \file */
/*
* kmp.h -- KPTS runtime header file.
*/
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef KMP_H
#define KMP_H
#include "kmp_config.h"
/* #define BUILD_PARALLEL_ORDERED 1 */
/* This fix replaces gettimeofday with clock_gettime for better scalability on
the Altix. Requires user code to be linked with -lrt. */
//#define FIX_SGI_CLOCK
/* Defines for OpenMP 3.0 tasking and auto scheduling */
#ifndef KMP_STATIC_STEAL_ENABLED
#define KMP_STATIC_STEAL_ENABLED 1
#endif
#define TASK_CURRENT_NOT_QUEUED 0
#define TASK_CURRENT_QUEUED 1
#ifdef BUILD_TIED_TASK_STACK
#define TASK_STACK_EMPTY 0 // entries when the stack is empty
#define TASK_STACK_BLOCK_BITS 5 // Used in TASK_STACK_SIZE and TASK_STACK_MASK
// Number of entries in each task stack array
#define TASK_STACK_BLOCK_SIZE (1 << TASK_STACK_BLOCK_BITS)
// Mask for determining index into stack block
#define TASK_STACK_INDEX_MASK (TASK_STACK_BLOCK_SIZE - 1)
#endif // BUILD_TIED_TASK_STACK
#define TASK_NOT_PUSHED 1
#define TASK_SUCCESSFULLY_PUSHED 0
#define TASK_TIED 1
#define TASK_UNTIED 0
#define TASK_EXPLICIT 1
#define TASK_IMPLICIT 0
#define TASK_PROXY 1
#define TASK_FULL 0
#define KMP_CANCEL_THREADS
#define KMP_THREAD_ATTR
// Android does not have pthread_cancel. Undefine KMP_CANCEL_THREADS if being
// built on Android
#if defined(__ANDROID__)
#undef KMP_CANCEL_THREADS
#endif
#include <signal.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* include <ctype.h> don't use; problems with /MD on Windows* OS NT due to bad
Microsoft library. Some macros provided below to replace these functions */
#ifndef __ABSOFT_WIN
#include <sys/types.h>
#endif
#include <limits.h>
#include <time.h>
#include <errno.h>
#include "kmp_os.h"
#include "kmp_safe_c_api.h"
#if KMP_STATS_ENABLED
class kmp_stats_list;
#endif
#if KMP_USE_HIER_SCHED
// Only include hierarchical scheduling if affinity is supported
#undef KMP_USE_HIER_SCHED
#define KMP_USE_HIER_SCHED KMP_AFFINITY_SUPPORTED
#endif
#if KMP_USE_HWLOC && KMP_AFFINITY_SUPPORTED
#include "hwloc.h"
#ifndef HWLOC_OBJ_NUMANODE
#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE
#endif
#ifndef HWLOC_OBJ_PACKAGE
#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET
#endif
#endif
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
#include <xmmintrin.h>
#endif
#include "kmp_debug.h"
#include "kmp_lock.h"
#include "kmp_version.h"
#if USE_DEBUGGER
#include "kmp_debugger.h"
#endif
#include "kmp_i18n.h"
#define KMP_HANDLE_SIGNALS (KMP_OS_UNIX || KMP_OS_WINDOWS)
#include "kmp_wrapper_malloc.h"
#if KMP_OS_UNIX
#include <unistd.h>
#if !defined NSIG && defined _NSIG
#define NSIG _NSIG
#endif
#endif
#if KMP_OS_LINUX
#pragma weak clock_gettime
#endif
#if OMPT_SUPPORT
#include "ompt-internal.h"
#endif
#if OMP_50_ENABLED
// Affinity format function
#include "kmp_str.h"
#endif
// 0 - no fast memory allocation, alignment: 8-byte on x86, 16-byte on x64.
// 3 - fast allocation using sync, non-sync free lists of any size, non-self
// free lists of limited size.
#ifndef USE_FAST_MEMORY
#define USE_FAST_MEMORY 3
#endif
#ifndef KMP_NESTED_HOT_TEAMS
#define KMP_NESTED_HOT_TEAMS 0
#define USE_NESTED_HOT_ARG(x)
#else
#if KMP_NESTED_HOT_TEAMS
#if OMP_40_ENABLED
#define USE_NESTED_HOT_ARG(x) , x
#else
// Nested hot teams feature depends on omp 4.0, disable it for earlier versions
#undef KMP_NESTED_HOT_TEAMS
#define KMP_NESTED_HOT_TEAMS 0
#define USE_NESTED_HOT_ARG(x)
#endif
#else
#define USE_NESTED_HOT_ARG(x)
#endif
#endif
// Assume using BGET compare_exchange instruction instead of lock by default.
#ifndef USE_CMP_XCHG_FOR_BGET
#define USE_CMP_XCHG_FOR_BGET 1
#endif
// Test to see if queuing lock is better than bootstrap lock for bget
// #ifndef USE_QUEUING_LOCK_FOR_BGET
// #define USE_QUEUING_LOCK_FOR_BGET
// #endif
#define KMP_NSEC_PER_SEC 1000000000L
#define KMP_USEC_PER_SEC 1000000L
/*!
@ingroup BASIC_TYPES
@{
*/
/*!
Values for bit flags used in the ident_t to describe the fields.
*/
enum {
/*! Use trampoline for internal microtasks */
KMP_IDENT_IMB = 0x01,
/*! Use c-style ident structure */
KMP_IDENT_KMPC = 0x02,
/* 0x04 is no longer used */
/*! Entry point generated by auto-parallelization */
KMP_IDENT_AUTOPAR = 0x08,
/*! Compiler generates atomic reduction option for kmpc_reduce* */
KMP_IDENT_ATOMIC_REDUCE = 0x10,
/*! To mark a 'barrier' directive in user code */
KMP_IDENT_BARRIER_EXPL = 0x20,
/*! To Mark implicit barriers. */
KMP_IDENT_BARRIER_IMPL = 0x0040,
KMP_IDENT_BARRIER_IMPL_MASK = 0x01C0,
KMP_IDENT_BARRIER_IMPL_FOR = 0x0040,
KMP_IDENT_BARRIER_IMPL_SECTIONS = 0x00C0,
KMP_IDENT_BARRIER_IMPL_SINGLE = 0x0140,
KMP_IDENT_BARRIER_IMPL_WORKSHARE = 0x01C0,
/*! To mark a static loop in OMPT callbacks */
KMP_IDENT_WORK_LOOP = 0x200,
/*! To mark a sections directive in OMPT callbacks */
KMP_IDENT_WORK_SECTIONS = 0x400,
/*! To mark a distirbute construct in OMPT callbacks */
KMP_IDENT_WORK_DISTRIBUTE = 0x800,
/*! Atomic hint; bottom four bits as omp_sync_hint_t. Top four reserved and
not currently used. If one day we need more bits, then we can use
an invalid combination of hints to mean that another, larger field
should be used in a different flag. */
KMP_IDENT_ATOMIC_HINT_MASK = 0xFF0000,
KMP_IDENT_ATOMIC_HINT_UNCONTENDED = 0x010000,
KMP_IDENT_ATOMIC_HINT_CONTENDED = 0x020000,
KMP_IDENT_ATOMIC_HINT_NONSPECULATIVE = 0x040000,
KMP_IDENT_ATOMIC_HINT_SPECULATIVE = 0x080000,
};
/*!
* The ident structure that describes a source location.
*/
typedef struct ident {
kmp_int32 reserved_1; /**< might be used in Fortran; see above */
kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; KMP_IDENT_KMPC
identifies this union member */
kmp_int32 reserved_2; /**< not really used in Fortran any more; see above */
#if USE_ITT_BUILD
/* but currently used for storing region-specific ITT */
/* contextual information. */
#endif /* USE_ITT_BUILD */
kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for C++ */
char const *psource; /**< String describing the source location.
The string is composed of semi-colon separated fields
which describe the source file, the function and a pair
of line numbers that delimit the construct. */
} ident_t;
/*!
@}
*/
// Some forward declarations.
typedef union kmp_team kmp_team_t;
typedef struct kmp_taskdata kmp_taskdata_t;
typedef union kmp_task_team kmp_task_team_t;
typedef union kmp_team kmp_team_p;
typedef union kmp_info kmp_info_p;
typedef union kmp_root kmp_root_p;
#ifdef __cplusplus
extern "C" {
#endif
/* ------------------------------------------------------------------------ */
/* Pack two 32-bit signed integers into a 64-bit signed integer */
/* ToDo: Fix word ordering for big-endian machines. */
#define KMP_PACK_64(HIGH_32, LOW_32) \
((kmp_int64)((((kmp_uint64)(HIGH_32)) << 32) | (kmp_uint64)(LOW_32)))
// Generic string manipulation macros. Assume that _x is of type char *
#define SKIP_WS(_x) \
{ \
while (*(_x) == ' ' || *(_x) == '\t') \
(_x)++; \
}
#define SKIP_DIGITS(_x) \
{ \
while (*(_x) >= '0' && *(_x) <= '9') \
(_x)++; \
}
#define SKIP_TOKEN(_x) \
{ \
while ((*(_x) >= '0' && *(_x) <= '9') || (*(_x) >= 'a' && *(_x) <= 'z') || \
(*(_x) >= 'A' && *(_x) <= 'Z') || *(_x) == '_') \
(_x)++; \
}
#define SKIP_TO(_x, _c) \
{ \
while (*(_x) != '\0' && *(_x) != (_c)) \
(_x)++; \
}
/* ------------------------------------------------------------------------ */
#define KMP_MAX(x, y) ((x) > (y) ? (x) : (y))
#define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
/* ------------------------------------------------------------------------ */
/* Enumeration types */
enum kmp_state_timer {
ts_stop,
ts_start,
ts_pause,
ts_last_state
};
enum dynamic_mode {
dynamic_default,
#ifdef USE_LOAD_BALANCE
dynamic_load_balance,
#endif /* USE_LOAD_BALANCE */
dynamic_random,
dynamic_thread_limit,
dynamic_max
};
/* external schedule constants, duplicate enum omp_sched in omp.h in order to
* not include it here */
#ifndef KMP_SCHED_TYPE_DEFINED
#define KMP_SCHED_TYPE_DEFINED
typedef enum kmp_sched {
kmp_sched_lower = 0, // lower and upper bounds are for routine parameter check
// Note: need to adjust __kmp_sch_map global array in case enum is changed
kmp_sched_static = 1, // mapped to kmp_sch_static_chunked (33)
kmp_sched_dynamic = 2, // mapped to kmp_sch_dynamic_chunked (35)
kmp_sched_guided = 3, // mapped to kmp_sch_guided_chunked (36)
kmp_sched_auto = 4, // mapped to kmp_sch_auto (38)
kmp_sched_upper_std = 5, // upper bound for standard schedules
kmp_sched_lower_ext = 100, // lower bound of Intel extension schedules
kmp_sched_trapezoidal = 101, // mapped to kmp_sch_trapezoidal (39)
#if KMP_STATIC_STEAL_ENABLED
kmp_sched_static_steal = 102, // mapped to kmp_sch_static_steal (44)
#endif
kmp_sched_upper,
kmp_sched_default = kmp_sched_static // default scheduling
} kmp_sched_t;
#endif
/*!
@ingroup WORK_SHARING
* Describes the loop schedule to be used for a parallel for loop.
*/
enum sched_type : kmp_int32 {
kmp_sch_lower = 32, /**< lower bound for unordered values */
kmp_sch_static_chunked = 33,
kmp_sch_static = 34, /**< static unspecialized */
kmp_sch_dynamic_chunked = 35,
kmp_sch_guided_chunked = 36, /**< guided unspecialized */
kmp_sch_runtime = 37,
kmp_sch_auto = 38, /**< auto */
kmp_sch_trapezoidal = 39,
/* accessible only through KMP_SCHEDULE environment variable */
kmp_sch_static_greedy = 40,
kmp_sch_static_balanced = 41,
/* accessible only through KMP_SCHEDULE environment variable */
kmp_sch_guided_iterative_chunked = 42,
kmp_sch_guided_analytical_chunked = 43,
/* accessible only through KMP_SCHEDULE environment variable */
kmp_sch_static_steal = 44,
#if OMP_45_ENABLED
/* static with chunk adjustment (e.g., simd) */
kmp_sch_static_balanced_chunked = 45,
kmp_sch_guided_simd = 46, /**< guided with chunk adjustment */
kmp_sch_runtime_simd = 47, /**< runtime with chunk adjustment */
#endif
/* accessible only through KMP_SCHEDULE environment variable */
kmp_sch_upper, /**< upper bound for unordered values */
kmp_ord_lower = 64, /**< lower bound for ordered values, must be power of 2 */
kmp_ord_static_chunked = 65,
kmp_ord_static = 66, /**< ordered static unspecialized */
kmp_ord_dynamic_chunked = 67,
kmp_ord_guided_chunked = 68,
kmp_ord_runtime = 69,
kmp_ord_auto = 70, /**< ordered auto */
kmp_ord_trapezoidal = 71,
kmp_ord_upper, /**< upper bound for ordered values */
#if OMP_40_ENABLED
/* Schedules for Distribute construct */
kmp_distribute_static_chunked = 91, /**< distribute static chunked */
kmp_distribute_static = 92, /**< distribute static unspecialized */
#endif
/* For the "nomerge" versions, kmp_dispatch_next*() will always return a
single iteration/chunk, even if the loop is serialized. For the schedule
types listed above, the entire iteration vector is returned if the loop is
serialized. This doesn't work for gcc/gcomp sections. */
kmp_nm_lower = 160, /**< lower bound for nomerge values */
kmp_nm_static_chunked =
(kmp_sch_static_chunked - kmp_sch_lower + kmp_nm_lower),
kmp_nm_static = 162, /**< static unspecialized */
kmp_nm_dynamic_chunked = 163,
kmp_nm_guided_chunked = 164, /**< guided unspecialized */
kmp_nm_runtime = 165,
kmp_nm_auto = 166, /**< auto */
kmp_nm_trapezoidal = 167,
/* accessible only through KMP_SCHEDULE environment variable */
kmp_nm_static_greedy = 168,
kmp_nm_static_balanced = 169,
/* accessible only through KMP_SCHEDULE environment variable */
kmp_nm_guided_iterative_chunked = 170,
kmp_nm_guided_analytical_chunked = 171,
kmp_nm_static_steal =
172, /* accessible only through OMP_SCHEDULE environment variable */
kmp_nm_ord_static_chunked = 193,
kmp_nm_ord_static = 194, /**< ordered static unspecialized */
kmp_nm_ord_dynamic_chunked = 195,
kmp_nm_ord_guided_chunked = 196,
kmp_nm_ord_runtime = 197,
kmp_nm_ord_auto = 198, /**< auto */
kmp_nm_ord_trapezoidal = 199,
kmp_nm_upper, /**< upper bound for nomerge values */
#if OMP_45_ENABLED
/* Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers. Since
we need to distinguish the three possible cases (no modifier, monotonic
modifier, nonmonotonic modifier), we need separate bits for each modifier.
The absence of monotonic does not imply nonmonotonic, especially since 4.5
says that the behaviour of the "no modifier" case is implementation defined
in 4.5, but will become "nonmonotonic" in 5.0.
Since we're passing a full 32 bit value, we can use a couple of high bits
for these flags; out of paranoia we avoid the sign bit.
These modifiers can be or-ed into non-static schedules by the compiler to
pass the additional information. They will be stripped early in the
processing in __kmp_dispatch_init when setting up schedules, so most of the
code won't ever see schedules with these bits set. */
kmp_sch_modifier_monotonic =
(1 << 29), /**< Set if the monotonic schedule modifier was present */
kmp_sch_modifier_nonmonotonic =
(1 << 30), /**< Set if the nonmonotonic schedule modifier was present */
#define SCHEDULE_WITHOUT_MODIFIERS(s) \
(enum sched_type)( \
(s) & ~(kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic))
#define SCHEDULE_HAS_MONOTONIC(s) (((s)&kmp_sch_modifier_monotonic) != 0)
#define SCHEDULE_HAS_NONMONOTONIC(s) (((s)&kmp_sch_modifier_nonmonotonic) != 0)
#define SCHEDULE_HAS_NO_MODIFIERS(s) \
(((s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)) == 0)
#else
/* By doing this we hope to avoid multiple tests on OMP_45_ENABLED. Compilers
can now eliminate tests on compile time constants and dead code that results
from them, so we can leave code guarded by such an if in place. */
#define SCHEDULE_WITHOUT_MODIFIERS(s) (s)
#define SCHEDULE_HAS_MONOTONIC(s) false
#define SCHEDULE_HAS_NONMONOTONIC(s) false
#define SCHEDULE_HAS_NO_MODIFIERS(s) true
#endif
kmp_sch_default = kmp_sch_static /**< default scheduling algorithm */
};
/* Type to keep runtime schedule set via OMP_SCHEDULE or omp_set_schedule() */
typedef union kmp_r_sched {
struct {
enum sched_type r_sched_type;
int chunk;
};
kmp_int64 sched;
} kmp_r_sched_t;
extern enum sched_type __kmp_sch_map[]; // map OMP 3.0 schedule types with our
// internal schedule types
enum library_type {
library_none,
library_serial,
library_turnaround,
library_throughput
};
#if KMP_OS_LINUX
enum clock_function_type {
clock_function_gettimeofday,
clock_function_clock_gettime
};
#endif /* KMP_OS_LINUX */
#if KMP_MIC_SUPPORTED
enum mic_type { non_mic, mic1, mic2, mic3, dummy };
#endif
/* -- fast reduction stuff ------------------------------------------------ */
#undef KMP_FAST_REDUCTION_BARRIER
#define KMP_FAST_REDUCTION_BARRIER 1
#undef KMP_FAST_REDUCTION_CORE_DUO
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
#define KMP_FAST_REDUCTION_CORE_DUO 1
#endif
enum _reduction_method {
reduction_method_not_defined = 0,
critical_reduce_block = (1 << 8),
atomic_reduce_block = (2 << 8),
tree_reduce_block = (3 << 8),
empty_reduce_block = (4 << 8)
};
// Description of the packed_reduction_method variable:
// The packed_reduction_method variable consists of two enum types variables
// that are packed together into 0-th byte and 1-st byte:
// 0: (packed_reduction_method & 0x000000FF) is a 'enum barrier_type' value of
// barrier that will be used in fast reduction: bs_plain_barrier or
// bs_reduction_barrier
// 1: (packed_reduction_method & 0x0000FF00) is a reduction method that will
// be used in fast reduction;
// Reduction method is of 'enum _reduction_method' type and it's defined the way
// so that the bits of 0-th byte are empty, so no need to execute a shift
// instruction while packing/unpacking
#if KMP_FAST_REDUCTION_BARRIER
#define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method, barrier_type) \
((reduction_method) | (barrier_type))
#define UNPACK_REDUCTION_METHOD(packed_reduction_method) \
((enum _reduction_method)((packed_reduction_method) & (0x0000FF00)))
#define UNPACK_REDUCTION_BARRIER(packed_reduction_method) \
((enum barrier_type)((packed_reduction_method) & (0x000000FF)))
#else
#define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method, barrier_type) \
(reduction_method)
#define UNPACK_REDUCTION_METHOD(packed_reduction_method) \
(packed_reduction_method)
#define UNPACK_REDUCTION_BARRIER(packed_reduction_method) (bs_plain_barrier)
#endif
#define TEST_REDUCTION_METHOD(packed_reduction_method, which_reduction_block) \
((UNPACK_REDUCTION_METHOD(packed_reduction_method)) == \
(which_reduction_block))
#if KMP_FAST_REDUCTION_BARRIER
#define TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER \
(PACK_REDUCTION_METHOD_AND_BARRIER(tree_reduce_block, bs_reduction_barrier))
#define TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER \
(PACK_REDUCTION_METHOD_AND_BARRIER(tree_reduce_block, bs_plain_barrier))
#endif
typedef int PACKED_REDUCTION_METHOD_T;
/* -- end of fast reduction stuff ----------------------------------------- */
#if KMP_OS_WINDOWS
#define USE_CBLKDATA
#if KMP_MSVC_COMPAT
#pragma warning(push)
#pragma warning(disable : 271 310)
#endif
#include <windows.h>
#if KMP_MSVC_COMPAT
#pragma warning(pop)
#endif
#endif
#if KMP_OS_UNIX
#include <dlfcn.h>
#include <pthread.h>
#endif
/* Only Linux* OS and Windows* OS support thread affinity. */
#if KMP_AFFINITY_SUPPORTED
// GROUP_AFFINITY is already defined for _MSC_VER>=1600 (VS2010 and later).
#if KMP_OS_WINDOWS
#if _MSC_VER < 1600 && KMP_MSVC_COMPAT
typedef struct GROUP_AFFINITY {
KAFFINITY Mask;
WORD Group;
WORD Reserved[3];
} GROUP_AFFINITY;
#endif /* _MSC_VER < 1600 */
#if KMP_GROUP_AFFINITY
extern int __kmp_num_proc_groups;
#else
static const int __kmp_num_proc_groups = 1;
#endif /* KMP_GROUP_AFFINITY */
typedef DWORD (*kmp_GetActiveProcessorCount_t)(WORD);
extern kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount;
typedef WORD (*kmp_GetActiveProcessorGroupCount_t)(void);
extern kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount;
typedef BOOL (*kmp_GetThreadGroupAffinity_t)(HANDLE, GROUP_AFFINITY *);
extern kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity;
typedef BOOL (*kmp_SetThreadGroupAffinity_t)(HANDLE, const GROUP_AFFINITY *,
GROUP_AFFINITY *);
extern kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity;
#endif /* KMP_OS_WINDOWS */
#if KMP_USE_HWLOC
extern hwloc_topology_t __kmp_hwloc_topology;
extern int __kmp_hwloc_error;
extern int __kmp_numa_detected;
extern int __kmp_tile_depth;
#endif
extern size_t __kmp_affin_mask_size;
#define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0)
#define KMP_AFFINITY_DISABLE() (__kmp_affin_mask_size = 0)
#define KMP_AFFINITY_ENABLE(mask_size) (__kmp_affin_mask_size = mask_size)
#define KMP_CPU_SET_ITERATE(i, mask) \
for (i = (mask)->begin(); (int)i != (mask)->end(); i = (mask)->next(i))
#define KMP_CPU_SET(i, mask) (mask)->set(i)
#define KMP_CPU_ISSET(i, mask) (mask)->is_set(i)
#define KMP_CPU_CLR(i, mask) (mask)->clear(i)
#define KMP_CPU_ZERO(mask) (mask)->zero()
#define KMP_CPU_COPY(dest, src) (dest)->copy(src)
#define KMP_CPU_AND(dest, src) (dest)->bitwise_and(src)
#define KMP_CPU_COMPLEMENT(max_bit_number, mask) (mask)->bitwise_not()
#define KMP_CPU_UNION(dest, src) (dest)->bitwise_or(src)
#define KMP_CPU_ALLOC(ptr) (ptr = __kmp_affinity_dispatch->allocate_mask())
#define KMP_CPU_FREE(ptr) __kmp_affinity_dispatch->deallocate_mask(ptr)
#define KMP_CPU_ALLOC_ON_STACK(ptr) KMP_CPU_ALLOC(ptr)
#define KMP_CPU_FREE_FROM_STACK(ptr) KMP_CPU_FREE(ptr)
#define KMP_CPU_INTERNAL_ALLOC(ptr) KMP_CPU_ALLOC(ptr)
#define KMP_CPU_INTERNAL_FREE(ptr) KMP_CPU_FREE(ptr)
#define KMP_CPU_INDEX(arr, i) __kmp_affinity_dispatch->index_mask_array(arr, i)
#define KMP_CPU_ALLOC_ARRAY(arr, n) \
(arr = __kmp_affinity_dispatch->allocate_mask_array(n))
#define KMP_CPU_FREE_ARRAY(arr, n) \
__kmp_affinity_dispatch->deallocate_mask_array(arr)
#define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) KMP_CPU_ALLOC_ARRAY(arr, n)
#define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) KMP_CPU_FREE_ARRAY(arr, n)
#define __kmp_get_system_affinity(mask, abort_bool) \
(mask)->get_system_affinity(abort_bool)
#define __kmp_set_system_affinity(mask, abort_bool) \
(mask)->set_system_affinity(abort_bool)
#define __kmp_get_proc_group(mask) (mask)->get_proc_group()
class KMPAffinity {
public:
class Mask {
public:
void *operator new(size_t n);
void operator delete(void *p);
void *operator new[](size_t n);
void operator delete[](void *p);
virtual ~Mask() {}
// Set bit i to 1
virtual void set(int i) {}
// Return bit i
virtual bool is_set(int i) const { return false; }
// Set bit i to 0
virtual void clear(int i) {}
// Zero out entire mask
virtual void zero() {}
// Copy src into this mask
virtual void copy(const Mask *src) {}
// this &= rhs
virtual void bitwise_and(const Mask *rhs) {}
// this |= rhs
virtual void bitwise_or(const Mask *rhs) {}
// this = ~this
virtual void bitwise_not() {}
// API for iterating over an affinity mask
// for (int i = mask->begin(); i != mask->end(); i = mask->next(i))
virtual int begin() const { return 0; }
virtual int end() const { return 0; }
virtual int next(int previous) const { return 0; }
// Set the system's affinity to this affinity mask's value
virtual int set_system_affinity(bool abort_on_error) const { return -1; }
// Set this affinity mask to the current system affinity
virtual int get_system_affinity(bool abort_on_error) { return -1; }
// Only 1 DWORD in the mask should have any procs set.
// Return the appropriate index, or -1 for an invalid mask.
virtual int get_proc_group() const { return -1; }
};
void *operator new(size_t n);
void operator delete(void *p);
// Need virtual destructor
virtual ~KMPAffinity() = default;
// Determine if affinity is capable
virtual void determine_capable(const char *env_var) {}
// Bind the current thread to os proc
virtual void bind_thread(int proc) {}
// Factory functions to allocate/deallocate a mask
virtual Mask *allocate_mask() { return nullptr; }
virtual void deallocate_mask(Mask *m) {}
virtual Mask *allocate_mask_array(int num) { return nullptr; }
virtual void deallocate_mask_array(Mask *m) {}
virtual Mask *index_mask_array(Mask *m, int index) { return nullptr; }
static void pick_api();
static void destroy_api();
enum api_type {
NATIVE_OS
#if KMP_USE_HWLOC
,
HWLOC
#endif
};
virtual api_type get_api_type() const {
KMP_ASSERT(0);
return NATIVE_OS;
}
private:
static bool picked_api;
};
typedef KMPAffinity::Mask kmp_affin_mask_t;
extern KMPAffinity *__kmp_affinity_dispatch;
// Declare local char buffers with this size for printing debug and info
// messages, using __kmp_affinity_print_mask().
#define KMP_AFFIN_MASK_PRINT_LEN 1024
enum affinity_type {
affinity_none = 0,
affinity_physical,
affinity_logical,
affinity_compact,
affinity_scatter,
affinity_explicit,
affinity_balanced,
affinity_disabled, // not used outsize the env var parser
affinity_default
};
enum affinity_gran {
affinity_gran_fine = 0,
affinity_gran_thread,
affinity_gran_core,
affinity_gran_tile,
affinity_gran_numa,
affinity_gran_package,
affinity_gran_node,
#if KMP_GROUP_AFFINITY
// The "group" granularity isn't necesssarily coarser than all of the
// other levels, but we put it last in the enum.
affinity_gran_group,
#endif /* KMP_GROUP_AFFINITY */
affinity_gran_default
};
enum affinity_top_method {
affinity_top_method_all = 0, // try all (supported) methods, in order
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
affinity_top_method_apicid,
affinity_top_method_x2apicid,
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
affinity_top_method_cpuinfo, // KMP_CPUINFO_FILE is usable on Windows* OS, too
#if KMP_GROUP_AFFINITY
affinity_top_method_group,
#endif /* KMP_GROUP_AFFINITY */
affinity_top_method_flat,
#if KMP_USE_HWLOC
affinity_top_method_hwloc,
#endif
affinity_top_method_default
};
#define affinity_respect_mask_default (-1)
extern enum affinity_type __kmp_affinity_type; /* Affinity type */
extern enum affinity_gran __kmp_affinity_gran; /* Affinity granularity */
extern int __kmp_affinity_gran_levels; /* corresponding int value */
extern int __kmp_affinity_dups; /* Affinity duplicate masks */
extern enum affinity_top_method __kmp_affinity_top_method;
extern int __kmp_affinity_compact; /* Affinity 'compact' value */
extern int __kmp_affinity_offset; /* Affinity offset value */
extern int __kmp_affinity_verbose; /* Was verbose specified for KMP_AFFINITY? */
extern int __kmp_affinity_warnings; /* KMP_AFFINITY warnings enabled ? */
extern int __kmp_affinity_respect_mask; // Respect process' init affinity mask?
extern char *__kmp_affinity_proclist; /* proc ID list */
extern kmp_affin_mask_t *__kmp_affinity_masks;
extern unsigned __kmp_affinity_num_masks;
extern void __kmp_affinity_bind_thread(int which);
extern kmp_affin_mask_t *__kmp_affin_fullMask;
extern char *__kmp_cpuinfo_file;
#endif /* KMP_AFFINITY_SUPPORTED */
#if OMP_40_ENABLED
// This needs to be kept in sync with the values in omp.h !!!
typedef enum kmp_proc_bind_t {
proc_bind_false = 0,
proc_bind_true,
proc_bind_master,
proc_bind_close,
proc_bind_spread,
proc_bind_intel, // use KMP_AFFINITY interface
proc_bind_default
} kmp_proc_bind_t;
typedef struct kmp_nested_proc_bind_t {
kmp_proc_bind_t *bind_types;
int size;
int used;
} kmp_nested_proc_bind_t;
extern kmp_nested_proc_bind_t __kmp_nested_proc_bind;
#endif /* OMP_40_ENABLED */
#if OMP_50_ENABLED
extern int __kmp_display_affinity;
extern char *__kmp_affinity_format;
static const size_t KMP_AFFINITY_FORMAT_SIZE = 512;
#endif // OMP_50_ENABLED
#if KMP_AFFINITY_SUPPORTED
#define KMP_PLACE_ALL (-1)
#define KMP_PLACE_UNDEFINED (-2)
// Is KMP_AFFINITY is being used instead of OMP_PROC_BIND/OMP_PLACES?
#define KMP_AFFINITY_NON_PROC_BIND \
((__kmp_nested_proc_bind.bind_types[0] == proc_bind_false || \
__kmp_nested_proc_bind.bind_types[0] == proc_bind_intel) && \
(__kmp_affinity_num_masks > 0 || __kmp_affinity_type == affinity_balanced))
#endif /* KMP_AFFINITY_SUPPORTED */
extern int __kmp_affinity_num_places;
#if OMP_40_ENABLED
typedef enum kmp_cancel_kind_t {
cancel_noreq = 0,
cancel_parallel = 1,
cancel_loop = 2,
cancel_sections = 3,
cancel_taskgroup = 4
} kmp_cancel_kind_t;
#endif // OMP_40_ENABLED
// KMP_HW_SUBSET support:
typedef struct kmp_hws_item {
int num;
int offset;
} kmp_hws_item_t;
extern kmp_hws_item_t __kmp_hws_socket;
extern kmp_hws_item_t __kmp_hws_node;
extern kmp_hws_item_t __kmp_hws_tile;
extern kmp_hws_item_t __kmp_hws_core;
extern kmp_hws_item_t __kmp_hws_proc;
extern int __kmp_hws_requested;
extern int __kmp_hws_abs_flag; // absolute or per-item number requested
/* ------------------------------------------------------------------------ */
#define KMP_PAD(type, sz) \
(sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))
// We need to avoid using -1 as a GTID as +1 is added to the gtid
// when storing it in a lock, and the value 0 is reserved.
#define KMP_GTID_DNE (-2) /* Does not exist */
#define KMP_GTID_SHUTDOWN (-3) /* Library is shutting down */
#define KMP_GTID_MONITOR (-4) /* Monitor thread ID */
#define KMP_GTID_UNKNOWN (-5) /* Is not known */
#define KMP_GTID_MIN (-6) /* Minimal gtid for low bound check in DEBUG */
#if OMP_50_ENABLED
/* OpenMP 5.0 Memory Management support */
extern int __kmp_memkind_available;
extern int __kmp_hbw_mem_available;
typedef void *omp_allocator_t;
extern const omp_allocator_t *OMP_NULL_ALLOCATOR;
extern const omp_allocator_t *omp_default_mem_alloc;
extern const omp_allocator_t *omp_large_cap_mem_alloc;
extern const omp_allocator_t *omp_const_mem_alloc;
extern const omp_allocator_t *omp_high_bw_mem_alloc;
extern const omp_allocator_t *omp_low_lat_mem_alloc;
extern const omp_allocator_t *omp_cgroup_mem_alloc;
extern const omp_allocator_t *omp_pteam_mem_alloc;
extern const omp_allocator_t *omp_thread_mem_alloc;
extern const omp_allocator_t *__kmp_def_allocator;
extern void __kmpc_set_default_allocator(int gtid, const omp_allocator_t *al);
extern const omp_allocator_t *__kmpc_get_default_allocator(int gtid);
extern void *__kmpc_alloc(int gtid, size_t sz, const omp_allocator_t *al);
extern void __kmpc_free(int gtid, void *ptr, const omp_allocator_t *al);
extern void __kmp_init_memkind();
extern void __kmp_fini_memkind();
#endif // OMP_50_ENABLED
/* ------------------------------------------------------------------------ */
#define KMP_UINT64_MAX \
(~((kmp_uint64)1 << ((sizeof(kmp_uint64) * (1 << 3)) - 1)))
#define KMP_MIN_NTH 1
#ifndef KMP_MAX_NTH
#if defined(PTHREAD_THREADS_MAX) && PTHREAD_THREADS_MAX < INT_MAX
#define KMP_MAX_NTH PTHREAD_THREADS_MAX
#else
#define KMP_MAX_NTH INT_MAX
#endif
#endif /* KMP_MAX_NTH */
#ifdef PTHREAD_STACK_MIN
#define KMP_MIN_STKSIZE PTHREAD_STACK_MIN
#else
#define KMP_MIN_STKSIZE ((size_t)(32 * 1024))
#endif
#define KMP_MAX_STKSIZE (~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))
#if KMP_ARCH_X86
#define KMP_DEFAULT_STKSIZE ((size_t)(2 * 1024 * 1024))
#elif KMP_ARCH_X86_64
#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))
#define KMP_BACKUP_STKSIZE ((size_t)(2 * 1024 * 1024))
#else
#define KMP_DEFAULT_STKSIZE ((size_t)(1024 * 1024))
#endif
#define KMP_DEFAULT_MALLOC_POOL_INCR ((size_t)(1024 * 1024))
#define KMP_MIN_MALLOC_POOL_INCR ((size_t)(4 * 1024))
#define KMP_MAX_MALLOC_POOL_INCR \
(~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))
#define KMP_MIN_STKOFFSET (0)
#define KMP_MAX_STKOFFSET KMP_MAX_STKSIZE
#if KMP_OS_DARWIN
#define KMP_DEFAULT_STKOFFSET KMP_MIN_STKOFFSET
#else
#define KMP_DEFAULT_STKOFFSET CACHE_LINE
#endif
#define KMP_MIN_STKPADDING (0)
#define KMP_MAX_STKPADDING (2 * 1024 * 1024)
#define KMP_BLOCKTIME_MULTIPLIER \
(1000) /* number of blocktime units per second */
#define KMP_MIN_BLOCKTIME (0)
#define KMP_MAX_BLOCKTIME \
(INT_MAX) /* Must be this for "infinite" setting the work */
#define KMP_DEFAULT_BLOCKTIME (200) /* __kmp_blocktime is in milliseconds */
#if KMP_USE_MONITOR
#define KMP_DEFAULT_MONITOR_STKSIZE ((size_t)(64 * 1024))
#define KMP_MIN_MONITOR_WAKEUPS (1) // min times monitor wakes up per second
#define KMP_MAX_MONITOR_WAKEUPS (1000) // max times monitor can wake up per sec
/* Calculate new number of monitor wakeups for a specific block time based on
previous monitor_wakeups. Only allow increasing number of wakeups */
#define KMP_WAKEUPS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \
(((blocktime) == KMP_MAX_BLOCKTIME) \
? (monitor_wakeups) \
: ((blocktime) == KMP_MIN_BLOCKTIME) \
? KMP_MAX_MONITOR_WAKEUPS \
: ((monitor_wakeups) > (KMP_BLOCKTIME_MULTIPLIER / (blocktime))) \
? (monitor_wakeups) \
: (KMP_BLOCKTIME_MULTIPLIER) / (blocktime))
/* Calculate number of intervals for a specific block time based on
monitor_wakeups */
#define KMP_INTERVALS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \
(((blocktime) + (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) - 1) / \
(KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)))
#else
#define KMP_BLOCKTIME(team, tid) \
(get__bt_set(team, tid) ? get__blocktime(team, tid) : __kmp_dflt_blocktime)
#if KMP_OS_UNIX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
// HW TSC is used to reduce overhead (clock tick instead of nanosecond).
extern kmp_uint64 __kmp_ticks_per_msec;
#if KMP_COMPILER_ICC
#define KMP_NOW() ((kmp_uint64)_rdtsc())
#else
#define KMP_NOW() __kmp_hardware_timestamp()
#endif
#define KMP_NOW_MSEC() (KMP_NOW() / __kmp_ticks_per_msec)
#define KMP_BLOCKTIME_INTERVAL(team, tid) \
(KMP_BLOCKTIME(team, tid) * __kmp_ticks_per_msec)
#define KMP_BLOCKING(goal, count) ((goal) > KMP_NOW())
#else
// System time is retrieved sporadically while blocking.
extern kmp_uint64 __kmp_now_nsec();
#define KMP_NOW() __kmp_now_nsec()
#define KMP_NOW_MSEC() (KMP_NOW() / KMP_USEC_PER_SEC)
#define KMP_BLOCKTIME_INTERVAL(team, tid) \
(KMP_BLOCKTIME(team, tid) * KMP_USEC_PER_SEC)
#define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())
#endif
#endif // KMP_USE_MONITOR
#define KMP_MIN_STATSCOLS 40
#define KMP_MAX_STATSCOLS 4096
#define KMP_DEFAULT_STATSCOLS 80
#define KMP_MIN_INTERVAL 0
#define KMP_MAX_INTERVAL (INT_MAX - 1)
#define KMP_DEFAULT_INTERVAL 0
#define KMP_MIN_CHUNK 1
#define KMP_MAX_CHUNK (INT_MAX - 1)
#define KMP_DEFAULT_CHUNK 1
#define KMP_DFLT_DISP_NUM_BUFF 7
#define KMP_MAX_ORDERED 8