Repository navigation
Expand file tree
/
Copy patherl_process.h
More file actions
2988 lines (2552 loc) · 97.2 KB
/
Copy patherl_process.h
File metadata and controls
2988 lines (2552 loc) · 97.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* %CopyrightBegin%
*
* Copyright Ericsson AB 1996-2023. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* %CopyrightEnd%
*/
#ifndef __PROCESS_H__
#define __PROCESS_H__
#include "sys.h"
#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
|| defined(ERL_PORT_TASK_C__) \
|| (ERTS_GLB_INLINE_INCL_FUNC_DEF \
&& defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */
#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
# define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
typedef struct process Process;
#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#include "erl_process_lock.h" /* Only pull out important types... */
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#define ERL_PORT_GET_PORT_TYPE_ONLY__
#include "erl_port.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#include "erl_vm.h"
#include "erl_message.h"
#include "erl_process_dict.h"
#include "erl_node_container_utils.h"
#include "erl_node_tables.h"
#include "erl_monitor_link.h"
#include "erl_hl_timer.h"
#include "erl_time.h"
#include "erl_atom_table.h"
#include "external.h"
#include "erl_mseg.h"
#include "erl_async.h"
#include "erl_gc.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#include "erl_trace.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS
#define ERTS_ONLY_SCHED_SPEC_ETS_DATA
#include "erl_db.h"
#undef ERTS_ONLY_SCHED_SPEC_ETS_DATA
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#include "erl_thr_progress.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT 0
#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_DEFAULT_MAX_PROCESSES (1 << 20)
#define ERTS_HEAP_ALLOC(Type, Size) \
erts_alloc((Type), (Size))
#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize) \
erts_realloc((Type), (Ptr), (NewSize))
#define ERTS_HEAP_FREE(Type, Ptr, Size) \
erts_free((Type), (Ptr))
#include "export.h"
struct saved_calls {
int len;
int n;
int cur;
Export *ct[1];
};
extern Export exp_send, exp_receive, exp_timeout;
extern int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
extern int ERTS_WRITE_UNLIKELY(erts_sched_balance_util);
extern Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
extern Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers);
extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers);
extern Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues);
extern int ERTS_WRITE_UNLIKELY(erts_no_aux_work_threads);
extern int erts_sched_thread_suggested_stack_size;
extern int erts_dcpu_sched_thread_suggested_stack_size;
extern int erts_dio_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20 /* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */
#include "erl_bits.h"
/* process priorities */
#define PRIORITY_MAX 0
#define PRIORITY_HIGH 1
#define PRIORITY_NORMAL 2
#define PRIORITY_LOW 3
#define ERTS_NO_PROC_PRIO_LEVELS 4
#define ERTS_NO_PROC_PRIO_QUEUES 3
#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)
#define ERTS_RUNQ_FLGS_PROCS_QMASK \
((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)
#define ERTS_RUNQ_FLGS_QMASK \
((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)
#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
(ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
(ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
(ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
(ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
(ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)
#define ERTS_RUNQ_FLG_BASE2 \
(ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLG_OUT_OF_WORK \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))
#define ERTS_RUNQ_FLG_MSB_EXEC \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 8))
#define ERTS_RUNQ_FLG_MISC_OP \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 9))
#define ERTS_RUNQ_FLG_HALTING \
(((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 10))
#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 12)
#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \
(ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
| ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
| ERTS_RUNQ_FLGS_EVACUATE_QMASK)
#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
(ERTS_RUNQ_FLG_INACTIVE \
| ERTS_RUNQ_FLG_OUT_OF_WORK \
| ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)
#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
(((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
(((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
(((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_RUNQ_FLGS_INIT(RQ, INIT) \
erts_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS) \
((Uint32) erts_atomic32_read_bor_relb(&(RQ)->flags, \
(erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS) \
((Uint32) erts_atomic32_read_bor_nob(&(RQ)->flags, \
(erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS) \
((Uint32) erts_atomic32_read_bset_relb(&(RQ)->flags, \
(erts_aint32_t) (MSK), \
(erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS) \
((Uint32) erts_atomic32_read_band_relb(&(RQ)->flags, \
(erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS) \
((Uint32) erts_atomic32_read_band_nob(&(RQ)->flags, \
(erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ) \
((Uint32) erts_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ) \
((Uint32) erts_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ) \
((Uint32) erts_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS) \
((Uint32) erts_atomic32_read_bset_relb(&(RQ)->flags, \
(erts_aint32_t) (MSK), \
(erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_POINTER_MASK (~((erts_aint_t) 3))
#define ERTS_RUNQ_BOUND_FLAG ((erts_aint_t) 1)
typedef enum {
ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED,
ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED,
ERTS_SCHDLR_SSPND_DONE,
ERTS_SCHDLR_SSPND_YIELD_RESTART,
ERTS_SCHDLR_SSPND_YIELD_DONE,
ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;
typedef enum {
ERTS_MIGRATE_SUCCESS,
ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;
#define ERTS_SSI_FLG_SLEEPING (((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING (((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING (((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING (((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED (((erts_aint32_t) 1) << 4)
#define ERTS_SSI_FLG_MSB_EXEC (((erts_aint32_t) 1) << 5)
#define ERTS_SSI_FLGS_MAX 6
#define ERTS_SSI_FLGS_SLEEP_TYPE \
(ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)
#define ERTS_SSI_FLGS_SLEEP \
(ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)
#define ERTS_SSI_FLGS_ALL \
(ERTS_SSI_FLGS_SLEEP \
| ERTS_SSI_FLG_WAITING \
| ERTS_SSI_FLG_SUSPENDED \
| ERTS_SSI_FLG_MSB_EXEC)
/*
* Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
* order relative each other. Most frequent at lowest at lowest
* index.
*
* ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
* highest index...
*
* Remember to update description in erts_pre_init_process()
* and etp-commands when adding new flags...
*/
typedef enum {
ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
ERTS_SSI_AUX_WORK_DD_IX,
ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
ERTS_SSI_AUX_WORK_MISC_IX,
ERTS_SSI_AUX_WORK_SET_TMO_IX,
ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
ERTS_SSI_AUX_WORK_YIELD_IX,
ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */
ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;
#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_YIELD \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_YIELD_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
(((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)
typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
typedef struct {
erts_mtx_t lock;
ErtsSchedulerSleepInfo *list; /* circular lifo list; points to last out */
} ErtsSchedulerSleepList;
struct ErtsSchedulerSleepInfo_ {
struct ErtsSchedulerData_ *esdp;
ErtsSchedulerSleepInfo *next;
ErtsSchedulerSleepInfo *prev;
erts_atomic32_t flags;
erts_tse_t *event;
struct erts_poll_thread *psi;
erts_atomic32_t aux_work;
};
/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW 8
#define ERTS_MAX_MISC_OPS 5
#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)
typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
union {
Eterm pid;
Process *p;
} u;
Uint64 started_interval;
ErtsProcList* next;
ErtsProcList* prev;
};
typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
ErtsMiscOpList *next;
void (*func)(void *arg);
void *arg;
};
typedef struct {
Process* first;
Process* last;
} ErtsRunPrioQueue;
typedef enum {
ERTS_SCHED_NORMAL = 0,
ERTS_SCHED_DIRTY_CPU = 1,
ERTS_SCHED_DIRTY_IO = 2,
ERTS_SCHED_TYPE_FIRST = ERTS_SCHED_NORMAL,
ERTS_SCHED_TYPE_LAST = ERTS_SCHED_DIRTY_IO
} ErtsSchedType;
typedef struct ErtsSchedulerData_ ErtsSchedulerData;
typedef struct ErtsRunQueue_ ErtsRunQueue;
typedef struct {
erts_atomic32_t len;
erts_aint32_t max_len;
int reds;
} ErtsRunQueueInfo;
#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
# undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
# define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
typedef erts_atomic64_t ErtsAtomicSchedTime;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
ErtsAtomicSchedTime last;
struct {
Uint64 short_interval;
Uint64 long_interval;
} worktime;
int is_working;
} ErtsRunQueueSchedUtil;
#endif
typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int sched_util;
#endif
Uint32 flags;
ErtsRunQueue *misc_evac_runq;
struct {
struct {
int here;
int other;
} limit;
ErtsRunQueue *runq;
Uint32 flags;
} prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;
typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;
struct ErtsMigrationPaths_ {
void *block;
ErtsMigrationPaths *next;
ErtsThrPrgrVal thr_prgr;
ErtsMigrationPath mpath[1];
};
struct ErtsRunQueue_ {
int ix;
erts_mtx_t mtx;
erts_cnd_t cnd;
ErtsSchedulerSleepList sleepers;
ErtsSchedulerData *scheduler;
int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
int woken;
erts_atomic32_t flags;
int check_balance_reds;
int full_reds_history_sum;
int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
int out_of_work_count;
erts_aint32_t max_len;
erts_atomic32_t len;
int wakeup_other;
int wakeup_other_reds;
struct {
Uint context_switches;
Uint reductions;
ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];
/* We use the same prio queue for low and
normal prio processes */
ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
} procs;
struct {
ErtsMiscOpList *start;
ErtsMiscOpList *end;
erts_atomic_t evac_runq;
} misc;
struct {
ErtsRunQueueInfo info;
Port *start;
Port *end;
} ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
ErtsRunQueueSchedUtil sched_util;
#endif
};
extern long erts_runq_supervision_interval;
typedef union {
ErtsRunQueue runq;
char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;
extern ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues);
#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do { \
(RQ)->procs.reductions += (AREDS); \
(RQ)->procs.prio_info[(PRIO)].reds += (REDS); \
(RQ)->check_balance_reds -= (REDS); \
(RQ)->wakeup_other_reds += (AREDS); \
(SD)->check_time_reds += (AREDS); \
} while (0)
#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS) \
do { \
(RQ)->ports.info.reds += (REDS); \
(RQ)->check_balance_reds -= (REDS); \
(RQ)->wakeup_other_reds += (REDS); \
(SD)->check_time_reds += (REDS); \
} while (0)
typedef struct {
union {
erts_atomic32_t mod; /* on dirty schedulers */
int need; /* "+sbu true" or scheduler_wall_time enabled */
} u;
int enabled;
Uint64 start;
struct {
Uint64 total;
Uint64 start;
} working;
} ErtsSchedWallTime;
typedef struct {
int sched;
erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;
typedef struct ErtsAuxWorkData_ {
int aux_work_tid;
ErtsThrAllocData alloc_data;
ErtsSchedulerData *esdp;
ErtsSchedulerSleepInfo *ssi;
ErtsThrPrgrVal current_thr_prgr;
ErtsThrPrgrVal latest_wakeup;
struct {
int ix;
ErtsThrPrgrVal thr_prgr;
} misc;
struct {
ErtsThrPrgrVal thr_prgr;
} dd;
struct {
ErtsThrPrgrVal thr_prgr;
} cncld_tmrs;
struct {
ErtsThrPrgrVal thr_prgr;
UWord size;
ErtsThrPrgrLaterOp *first;
ErtsThrPrgrLaterOp *last;
Uint list_len;
} later_op;
struct {
int need_thr_prgr;
ErtsThrPrgrVal thr_prgr;
void *queue;
} async_ready;
struct {
Uint64 next;
int *sched2jix;
int jix;
ErtsDelayedAuxWorkWakeupJob *job;
} delayed_wakeup;
struct {
ErtsAlcuBlockscanYieldData alcu_blockscan;
ErtsEtsAllYieldData ets_all;
/* Other yielding operations... */
} yield;
struct {
struct {
erts_aint32_t flags;
void (*callback)(void *);
void *arg;
} wait_completed;
} debug;
} ErtsAuxWorkData;
#define ERTS_SCHED_AUX_YIELD_DATA(ESDP, NAME) \
(&(ESDP)->aux_work_data.yield.NAME)
void erts_more_yield_aux_work(ErtsAuxWorkData *);
ErtsAuxWorkData *erts_get_aux_work_data(void);
typedef enum {
ERTS_DIRTY_CPU_SCHEDULER,
ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;
typedef struct ErtsSchedulerRegisters_ {
union {
struct aux_regs__ {
#ifdef BEAMASM
/* On normal schedulers we allocate this structure on the "C stack"
* to allow stack switching without needing to read memory or
* occupy a register; we simply compute the stack address from the
* register pointer.
*
* This is placed first because the stack grows downwards.
*
* In special builds that don't execute native code on the Erlang
* stack (e.g. `valgrind`), this will instead hold the original
* thread stack pointer when executing code that requires a certain
* stack alignment. */
UWord runtime_stack[1];
#ifdef ERTS_MSACC_EXTENDED_STATES
ErtsMsAcc *erts_msacc_cache;
#endif
/* Temporary memory used by beamasm for allocations within
* instructions */
UWord TMP_MEM[5];
#endif
/* erl_bits.c state */
struct erl_bits_state erl_bits_state;
} d;
char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct aux_regs__))];
} aux_regs;
union {
Eterm d[ERTS_X_REGS_ALLOCATED];
char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Eterm[ERTS_X_REGS_ALLOCATED]))];
} x_reg_array;
union {
FloatDef d[MAX_REG];
char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(FloatDef[MAX_REG]))];
} f_reg_array;
#ifdef BEAMASM
/* Seldom-used scheduler-specific data. */
ErtsCodePtr start_time_i;
UWord start_time;
#if (!defined(NATIVE_ERLANG_STACK) || defined(__aarch64__)) && defined(JIT_HARD_DEBUG)
/* Holds the initial thread stack pointer. Used to ensure that everything
* that is pushed to the stack is also popped. */
UWord *initial_sp;
#elif defined(NATIVE_ERLANG_STACK) && defined(DEBUG) && !defined(__aarch64__)
/* Raw pointers to the start and end of the stack. Used to test bounds
* without clobbering any registers. */
UWord *runtime_stack_start;
UWord *runtime_stack_end;
#endif
#endif
} ErtsSchedulerRegisters;
struct ErtsSchedulerData_ {
ErtsSchedulerRegisters *registers;
ErtsTimerWheel *timer_wheel;
ErtsNextTimeoutRef next_tmo_ref;
ErtsHLTimerService *timer_service;
ethr_tid tid; /* Thread id */
void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
Process *free_process;
ErtsThrPrgrData thr_progress_data;
ErtsSchedulerSleepInfo *ssi;
Process *current_process;
ErtsSchedType type;
Uint no; /* Scheduler number for normal schedulers */
Uint dirty_no; /* Scheduler number for dirty schedulers */
int flxctr_slot_no; /* slot nr when a flxctr is used */
struct enif_environment_t *current_nif;
Process *dirty_shadow_process;
Port *current_port;
ErtsRunQueue *run_queue;
int virtual_reds;
int cpu_id; /* >= 0 when bound */
ErtsAuxWorkData aux_work_data;
ErtsAtomCacheMap atom_cache_map;
ErtsMonotonicTime last_monotonic_time;
int check_time_reds;
Uint32 thr_id;
Uint64 unique;
Uint64 ref;
struct {
Uint64 out;
Uint64 in;
} io;
struct {
ErtsSignal* sig;
Eterm to;
#ifdef DEBUG
Process* dbg_from;
#endif
} pending_signal;
Uint64 reductions;
Uint64 rand_state;
ErtsSchedWallTime sched_wall_time;
ErtsGCInfo gc_info;
ErtsPortTaskHandle nosuspend_port_task_handle;
union {
ErtsEtsTables ets_tables;
erts_atomic32_t dirty_nif_halt_info;
} u;
#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
erts_alloc_verify_func_t verify_unused_temp_alloc;
Allctr_t *verify_unused_temp_alloc_data;
#endif
};
typedef union {
ErtsSchedulerData esd;
char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;
extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data);
extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data);
extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data);
#if defined(ERTS_ENABLE_LOCK_CHECK)
int erts_lc_runq_is_locked(ErtsRunQueue *);
#endif
void
erts_debug_later_op_foreach(void (*callback)(void*),
void (*func)(void *, ErtsThrPrgrVal, void *),
void *arg);
void
erts_debug_free_process_foreach(void (*func)(Process *, void *), void *arg);
void
erts_debug_proc_monitor_link_foreach(Process *proc,
int (*monitor_func)(ErtsMonitor *, void *, Sint ),
int (*link_func)(ErtsLink *, void *, Sint ),
void *arg);
#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
/*
* Run queue locked during modifications. We use atomic ops since
* other threads peek at values without run queue lock.
*/
ERTS_GLB_INLINE void erts_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
erts_aint32_t len;
ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
len = erts_atomic32_read_dirty(&rq->len);
if (len == 0)
erts_non_empty_runq(rq);
len++;
if (rq->max_len < len)
rq->max_len = len;
ASSERT(len > 0);
erts_atomic32_set_nob(&rq->len, len);
len = erts_atomic32_read_dirty(&rqi->len);
ASSERT(len >= 0);
if (len == 0) {
ASSERT((erts_atomic32_read_nob(&rq->flags)
& ((erts_aint32_t) (1 << prio))) == 0);
erts_atomic32_read_bor_nob(&rq->flags,
(erts_aint32_t) (1 << prio));
}
len++;
if (rqi->max_len < len)
rqi->max_len = len;
erts_atomic32_set_relb(&rqi->len, len);
}
ERTS_GLB_INLINE void
erts_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
erts_aint32_t len;
ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
len = erts_atomic32_read_dirty(&rq->len);
len--;
ASSERT(len >= 0);
erts_atomic32_set_nob(&rq->len, len);
len = erts_atomic32_read_dirty(&rqi->len);
len--;
ASSERT(len >= 0);
if (len == 0) {
ASSERT((erts_atomic32_read_nob(&rq->flags)
& ((erts_aint32_t) (1 << prio))));
erts_atomic32_read_band_nob(&rq->flags,
~((erts_aint32_t) (1 << prio)));
}
erts_atomic32_set_relb(&rqi->len, len);
}
ERTS_GLB_INLINE void
erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
erts_aint32_t len;
ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
len = erts_atomic32_read_dirty(&rqi->len);
ASSERT(rqi->max_len >= len);
rqi->max_len = len;
}
#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
#define RUNQ_READ_LEN(X) erts_atomic32_read_nob((X))
#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
/*
* Process Specific Data.
*
* NOTE: Only use PSD for very rarely used data.
*/
#define ERTS_PSD_ERROR_HANDLER 0
#define ERTS_PSD_SAVED_CALLS_BUF 1
#define ERTS_PSD_SCHED_ID 2
#define ERTS_PSD_CALL_TIME_BP 3
#define ERTS_PSD_DELAYED_GC_TASK_QS 4
#define ERTS_PSD_NFUNC_TRAP_WRAPPER 5
#define ERTS_PSD_ETS_OWNED_TABLES 6
#define ERTS_PSD_ETS_FIXED_TABLES 7
#define ERTS_PSD_DIST_ENTRY 8
#define ERTS_PSD_PENDING_SUSPEND 9
#define ERTS_PSD_CALL_MEMORY_BP 10
#define ERTS_PSD_SIZE 11
typedef struct {
void *data[ERTS_PSD_SIZE];
} ErtsPSD;
#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)
#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ((ErtsProcLocks) 0)
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ((ErtsProcLocks) 0)
#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NFUNC_TRAP_WRAPPER_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NFUNC_TRAP_WRAPPER_SET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_ETS_OWNED_TABLES_SET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_LC_PSD_ANY_LOCK
#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCKS_ALL
#define ERTS_PSD_PENDING_SUSPEND_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_PENDING_SUSPEND_SET_LOCKS ERTS_PROC_LOCK_MAIN
typedef struct {
ErtsProcLocks get_locks;
ErtsProcLocks set_locks;
} ErtsLcPSDLocks;
extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
#endif
#define ERTS_SCHED_STAT_MODIFY_DISABLE 1
#define ERTS_SCHED_STAT_MODIFY_ENABLE 2
#define ERTS_SCHED_STAT_MODIFY_CLEAR 3
typedef struct {
erts_spinlock_t lock;
int enabled;
struct {
Eterm name;
Uint total_executed;
Uint executed;
Uint total_migrated;
Uint migrated;
} prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;
extern erts_sched_stat_t erts_sched_stat;
typedef struct {
Eterm reason;
ErlHeapFragment *bp;
} ErtsPendExit;
typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;
/* Defines to ease the change of memory architecture */
# define HEAP_START(p) (p)->heap
# define HEAP_TOP(p) (p)->htop
/* The redzone is reserved for Erlang code and runtime functions may not use it
* on its own, but it's okay for them to run when the redzone is used.
*
* Therefore, we set the heap limit to HTOP or the start of the redzone,
* whichever is higher. */
# define HEAP_LIMIT(p) \
(ASSERT((p)->htop <= (p)->stop), \
MAX((p)->htop, (p)->stop - S_REDZONE))
#ifdef ERLANG_FRAME_POINTERS
/* The current frame pointer on the Erlang stack. */
# define FRAME_POINTER(p) (p)->frame_pointer
#else
/* We define this to a trapping lvalue when frame pointers are unsupported to
* provoke crashes when used without checking `erts_frame_layout`. The checks
* will always be optimized out because the variable is hardcoded to
* `ERTS_FRAME_LAYOUT_RA`. */
# define FRAME_POINTER(p) (((Eterm ** volatile)0xbadf00d)[0])
# ifndef erts_frame_layout
# error "erts_frame_layout has not been hardcoded to ERTS_FRAME_LAYOUT_RA"
# endif
#endif
# define HEAP_END(p) (p)->hend
# define HEAP_SIZE(p) (p)->heap_sz
# define STACK_START(p) (p)->hend
# define STACK_TOP(p) (p)->stop
# define STACK_END(p) (p)->htop
# define HIGH_WATER(p) (p)->high_water
# define OLD_HEND(p) (p)->old_hend
# define OLD_HTOP(p) (p)->old_htop
# define OLD_HEAP(p) (p)->old_heap
# define GEN_GCS(p) (p)->gen_gcs
# define MAX_GEN_GCS(p) (p)->max_gen_gcs
# define FLAGS(p) (p)->flags
# define MBUF(p) (p)->mbuf
# define MBUF_SIZE(p) (p)->mbuf_sz
# define MSO(p) (p)->off_heap
# define MIN_HEAP_SIZE(p) (p)->min_heap_size
# define MAX_HEAP_SIZE_GET(p) ((p)->max_heap_size >> 3)
# define MAX_HEAP_SIZE_SET(p, sz) ((p)->max_heap_size = ((sz) << 3) | \
MAX_HEAP_SIZE_FLAGS_GET(p))
# define MAX_HEAP_SIZE_FLAGS_GET(p) ((p)->max_heap_size & 0x7)