-
Notifications
You must be signed in to change notification settings - Fork 0
/
map.go
1637 lines (1528 loc) · 59.2 KB
/
map.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime
// This file contains the implementation of Go's map type.
//
// A map is just a hash table. The data is arranged
// into an array of buckets. Each bucket contains up to
// 8 key/value pairs. The low-order bits of the hash are
// used to select a bucket. Each bucket contains a few
// high-order bits of each hash to distinguish the entries
// within a single bucket.
//
// If more than 8 keys hash to a bucket, we chain on
// extra buckets.
//
// When the hashtable grows, we allocate a new array
// of buckets twice as big. Buckets are incrementally
// copied from the old bucket array to the new bucket array.
//
// Map iterators walk through the array of buckets and
// return the keys in walk order (bucket #, then overflow
// chain order, then bucket index). To maintain iteration
// semantics, we never move keys within their bucket (if
// we did, keys might be returned 0 or 2 times). When
// growing the table, iterators remain iterating through the
// old table and must check the new table if the bucket
// they are iterating through has been moved ("evacuated")
// to the new table.
// Picking loadFactor: too large and we have lots of overflow
// buckets, too small and we waste a lot of space. I wrote
// a simple program to check some stats for different loads:
// (64-bit, 8 byte keys and values)
// loadFactor %overflow bytes/entry hitprobe missprobe
// 4.00 2.13 20.77 3.00 4.00
// 4.50 4.05 17.30 3.25 4.50
// 5.00 6.85 14.77 3.50 5.00
// 5.50 10.55 12.94 3.75 5.50
// 6.00 15.27 11.67 4.00 6.00
// 6.50 20.90 10.79 4.25 6.50
// 7.00 27.14 10.15 4.50 7.00
// 7.50 34.03 9.73 4.75 7.50
// 8.00 41.10 9.40 5.00 8.00
//
// %overflow = percentage of buckets which have an overflow bucket
// bytes/entry = overhead bytes used per key/value pair
// hitprobe = # of entries to check when looking up a present key
// missprobe = # of entries to check when looking up an absent key
//
// Keep in mind this data is for maximally loaded tables, i.e. just
// before the table grows. Typical tables will be somewhat less loaded.
import (
"runtime/internal/atomic"
"runtime/internal/math"
"runtime/internal/sys"
"unsafe"
)
const (
// Maximum number of key/value pairs a bucket can hold.
bucketCntBits = 3
bucketCnt = 1 << bucketCntBits // 哈希桶的数量 8
// Maximum average load of a bucket that triggers growth is 6.5.
// Represent as loadFactorNum/loadFactDen, to allow integer math.
// 最大的负载因子为 6.5 ,转换为 loadFactorNum/loadFactDen ,来支持整型计算
loadFactorNum = 13
loadFactorDen = 2
// Maximum key or value size to keep inline (instead of mallocing per element).
// Must fit in a uint8.
// Fast versions cannot handle big values - the cutoff size for
// fast versions in cmd/compile/internal/gc/walk.go must be at most this value.
// key value 能够内联的最大内存大小(而不是为每个元素分配)
maxKeySize = 128
maxValueSize = 128
// data offset should be the size of the bmap struct, but needs to be
// aligned correctly. For amd64p32 this means 64-bit alignment
// even though pointers are 32 bit.
// 数据的偏移
dataOffset = unsafe.Offsetof(struct {
b bmap
v int64
}{}.v)
// Possible tophash values. We reserve a few possibilities for special marks.
// Each bucket (including its overflow buckets, if any) will have either all or none of its
// entries in the evacuated* states (except during the evacuate() method, which only happens
// during map writes and thus no one else can observe the map during that time).
// tophash 可能的值,以下是一些特殊标记,只有在 evacuate() 函数期间才有 evacuated* 值
// emptyRest 此单元格为空,并且高索引和overflows也没有非空单元格可
// emptyOne 此单元格为空
// evacuatedX 此单元格已经迁移了,进行的是平移迁移,也就是迁移到相同的下标下
// evacuatedY 此单元格已经迁移了,进行的是向后迁移,也就是迁移到相同的下标后面
// evacuatedEmpty 此单元格已经迁移了,原来的没有 emptyOne
// minTopHash tophash 最小的值
emptyRest = 0 // this cell is empty, and there are no more non-empty cells at higher indexes or overflows.
emptyOne = 1 // this cell is empty
evacuatedX = 2 // key/value is valid. Entry has been evacuated to first half of larger table.
evacuatedY = 3 // same as above, but evacuated to second half of larger table.
evacuatedEmpty = 4 // cell is empty, bucket is evacuated.
minTopHash = 5 // minimum tophash for a normal filled cell.
// flags
iterator = 1 // there may be an iterator using buckets // 可能在迭代 buckets
oldIterator = 2 // there may be an iterator using oldbuckets // 可能在迭代 oldbuckets
hashWriting = 4 // a goroutine is writing to the map // 有 goroutine 在写 map
sameSizeGrow = 8 // the current map growth is to a new map of the same size // 相同大小的扩容,扩容前后哈希桶数量相等,空的k/v太多了,做清理
// sentinel bucket ID for iterator checks
noCheck = 1<<(8*sys.PtrSize) - 1 // 哨兵标志
)
// isEmpty reports whether the given tophash array entry represents an empty bucket entry.
// isEmpty 返回 tophash 是否为空
func isEmpty(x uint8) bool {
return x <= emptyOne
}
// A header for a Go map.
// GO map 头信息
type hmap struct {
// Note: the format of the hmap is also encoded in cmd/compile/internal/gc/reflect.go.
// Make sure this stays in sync with the compiler's definition.
count int // # live cells == size of map. Must be first (used by len() builtin) // Map中元素数量
flags uint8 // // 读、写、扩容、迭代等标记,用于记录map当前状态
B uint8 // log_2 of # of buckets (can hold up to loadFactor * 2^B items) // 2^B 为桶的数量,(1<< B * 6.5)为最多元素的数量
noverflow uint16 // approximate number of overflow buckets; see incrnoverflow for details // 溢出桶个数,当溢出桶个数过多时,这个值是一个近似值
hash0 uint32 // hash seed // 计算key哈希值的随机值,保证一个key在不同map中存放的位置是随机的
buckets unsafe.Pointer // array of 2^B Buckets. may be nil if count==0. // 桶指针
oldbuckets unsafe.Pointer // previous bucket array of half the size, non-nil only when growing // 桶指针(只有扩容的时候才使用,指向旧的桶)
nevacuate uintptr // progress counter for evacuation (buckets less than this have been evacuated) // 用于桶迁移,已迁移哈希桶个数
extra *mapextra // optional fields // 当 bucket 中元素超过8个元素,通过 extra 来扩展 bucket
}
// mapextra holds fields that are not present on all maps.
// mapextra 记录不在 maps 中的字段
type mapextra struct {
// If both key and value do not contain pointers and are inline, then we mark bucket
// type as containing no pointers. This avoids scanning such maps.
// However, bmap.overflow is a pointer. In order to keep overflow buckets
// alive, we store pointers to all overflow buckets in hmap.extra.overflow and hmap.extra.oldoverflow.
// overflow and oldoverflow are only used if key and value do not contain pointers.
// overflow contains overflow buckets for hmap.buckets.
// oldoverflow contains overflow buckets for hmap.oldbuckets.
// 如果 key 和 value 都不包含指针,并且可以 inline (<=128字节),那么就标记 bucket(bmap) 为没有指针的类型,这样就可以
// 避免 GC 扫描整个 map 。然而 bmap.overflow 也是一个指针,所以就将其放到 hmap.extra.overflow 和 hmap.extra.oldoverflow
// 中,来保持引用,不会被 GC 回收了。
// 当kev/value不为指针时,溢出桶存放到mapextra结构中,overflow存放buckets中的溢出桶,oldoverflow存放oldbuckets中的溢出桶
overflow *[]*bmap // 以切片形式存放buckets中的每个溢出桶
oldoverflow *[]*bmap // 以切片形式存放oldbuckets中的每个溢出桶
// nextOverflow holds a pointer to a free overflow bucket.
// nextOverflow 指向空闲溢出桶 ,预分配的
nextOverflow *bmap
}
// A bucket for a Go map.
// Go Map 桶结构
type bmap struct {
// tophash generally contains the top byte of the hash value
// for each key in this bucket. If tophash[0] < minTopHash,
// tophash[0] is a bucket evacuation state instead.
// tophash 通常包含 hash 值的高 8 位 , 如果 tophash[0] < minTopHash ,则 tophash[0] 是迁移状态
tophash [bucketCnt]uint8 // bucketCnt = 8
// Followed by bucketCnt keys and then bucketCnt values.
// NOTE: packing all the keys together and then all the values together makes the
// code a bit more complicated than alternating key/value/key/value/... but it allows
// us to eliminate padding which would be needed for, e.g., map[int64]int8.
// Followed by an overflow pointer.
// 接下来就是 bucketCnt 个 keys,再就是 bucketCnt 个 values。
// 所有的 keys 一起,所有的 value 在一起,而不是 key/value 相间,可以消除填充。
// 后面再跟一个溢出指针 overflow ,overflow 组成链
//
// tophash[0] ... tophash[7] key[0] ... key[7] value[0] ... value[7] [overflow *bmap]
}
// A hash iteration structure.
// If you modify hiter, also change cmd/compile/internal/gc/reflect.go to indicate
// the layout of this structure.
// 迭代器
type hiter struct {
// key, value 必须在第一和第二个字段,在 cmd/compile/internal/gc/range.go 中会这么来获取 key value
key unsafe.Pointer // Must be in first position. Write nil to indicate iteration end (see cmd/internal/gc/range.go).
value unsafe.Pointer // Must be in second position (see cmd/internal/gc/range.go).
t *maptype // map 类型
h *hmap // 对应的 map
buckets unsafe.Pointer // bucket ptr at hash_iter initialization time // 初始化的时候设置的 buckets
bptr *bmap // current bucket // 当前的 bucket
overflow *[]*bmap // keeps overflow buckets of hmap.buckets alive // 让 hmap.buckets alive
oldoverflow *[]*bmap // keeps overflow buckets of hmap.oldbuckets alive // 让 hmap.buckets alive
startBucket uintptr // bucket iteration started at // 开始的 bucket
offset uint8 // intra-bucket offset to start from during iteration (should be big enough to hold bucketCnt-1) // bucket 内偏移
wrapped bool // already wrapped around from end of bucket array to beginning // 是否以及从最后遍历到前面来了,以及绕回来了
B uint8 // B
i uint8 // i
bucket uintptr // bucket 位置
checkBucket uintptr // checkBucket
}
// bucketShift returns 1<<b, optimized for code generation.
func bucketShift(b uint8) uintptr {
if sys.GoarchAmd64|sys.GoarchAmd64p32|sys.Goarch386 != 0 {
b &= sys.PtrSize*8 - 1 // help x86 archs remove shift overflow checks
}
return uintptr(1) << b
}
// bucketMask returns 1<<b - 1, optimized for code generation.
func bucketMask(b uint8) uintptr {
return bucketShift(b) - 1
}
// tophash calculates the tophash value for hash.
// tophash 计算 hash值的 tophash
func tophash(hash uintptr) uint8 {
// 取高 8 位作为 tophash
top := uint8(hash >> (sys.PtrSize*8 - 8))
// 如果小于 minTopHas 则加上 minTopHash ,小于 minTopHash 为特殊标记
if top < minTopHash {
top += minTopHash
}
return top
}
// evacuated 判断是否已经迁移
func evacuated(b *bmap) bool {
h := b.tophash[0]
return h > emptyOne && h < minTopHash
}
// 返回 bmap 的溢出指针
func (b *bmap) overflow(t *maptype) *bmap {
// 溢出指针在 bmap 的最后面
return *(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-sys.PtrSize))
}
// 设置 bmap 的溢出指针
func (b *bmap) setoverflow(t *maptype, ovf *bmap) {
*(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-sys.PtrSize)) = ovf
}
// 返回 bmap 的 keys 头指针
func (b *bmap) keys() unsafe.Pointer {
return add(unsafe.Pointer(b), dataOffset)
}
// incrnoverflow increments h.noverflow.
// noverflow counts the number of overflow buckets.
// This is used to trigger same-size map growth.
// See also tooManyOverflowBuckets.
// To keep hmap small, noverflow is a uint16.
// When there are few buckets, noverflow is an exact count.
// When there are many buckets, noverflow is an approximate count.
// incrnoverflow 增加 h.noverflow 。noverflow 统计溢出桶个数。用来触发相同大小的 map 扩容。
// 当桶小的时候,noverflow 是精确值;当 桶很多的时候,noverflow 是近似值。
func (h *hmap) incrnoverflow() {
// We trigger same-size map growth if there are
// as many overflow buckets as buckets.
// We need to be able to count to 1<<h.B.
// 如果溢出桶和桶的数量一样,则触发相同大小的 map 扩容。我们需要统计到 1<<h.B 。
// 如果 h.B < 16 ,则直接 noverflow++ 。
if h.B < 16 {
h.noverflow++
return
}
// Increment with probability 1/(1<<(h.B-15)).
// When we reach 1<<15 - 1, we will have approximately
// as many overflow buckets as buckets.
// 增加的概率为: 1/(1<<(h.B-15)) 。当达到 1<<15 - 1 ,将有大约和桶一样多的溢出桶。
mask := uint32(1)<<(h.B-15) - 1
// Example: if h.B == 18, then mask == 7,
// and fastrand & 7 == 0 with probability 1/8.
if fastrand()&mask == 0 {
h.noverflow++
}
}
// 新建溢出桶
func (h *hmap) newoverflow(t *maptype, b *bmap) *bmap {
var ovf *bmap
if h.extra != nil && h.extra.nextOverflow != nil {
// We have preallocated overflow buckets available.
// See makeBucketArray for more details.
// 有预分配的溢出桶
ovf = h.extra.nextOverflow
if ovf.overflow(t) == nil {
// We're not at the end of the preallocated overflow buckets. Bump the pointer.
// 不是最后一个预分配的溢出桶,将后续的设置到 h.extra.nextOverflow
// makeBucketArray 中可以看到,创建的溢出桶中,只有最后一个设置了 overflow ,所以如果为 nil 表示不是最后一个
h.extra.nextOverflow = (*bmap)(add(unsafe.Pointer(ovf), uintptr(t.bucketsize)))
} else {
// This is the last preallocated overflow bucket.
// Reset the overflow pointer on this bucket,
// which was set to a non-nil sentinel value.
// 这个是最后预分配的溢出桶,
ovf.setoverflow(t, nil)
h.extra.nextOverflow = nil
}
} else {
// 没有预分配的溢出桶, 则新建一个
ovf = (*bmap)(newobject(t.bucket))
}
// 增加 noverflow
h.incrnoverflow()
// 如果不是指针,则加入到 h.extra.overflow
if t.bucket.kind&kindNoPointers != 0 {
h.createOverflow()
*h.extra.overflow = append(*h.extra.overflow, ovf)
}
b.setoverflow(t, ovf)
return ovf
}
// createOverflow 创建溢出桶,初始化下 h.extra.overflow
func (h *hmap) createOverflow() {
if h.extra == nil {
h.extra = new(mapextra)
}
if h.extra.overflow == nil {
h.extra.overflow = new([]*bmap)
}
}
// make map
func makemap64(t *maptype, hint int64, h *hmap) *hmap {
if int64(int(hint)) != hint {
hint = 0
}
return makemap(t, int(hint), h)
}
// makehmap_small implements Go map creation for make(map[k]v) and
// make(map[k]v, hint) when hint is known to be at most bucketCnt
// at compile time and the map needs to be allocated on the heap.
// makehmap_small 实现 make(map[k]v) 和 (map[k]v, hint) hint<=bucketCnt , 并不初始化桶。
//
func makemap_small() *hmap {
h := new(hmap)
h.hash0 = fastrand() // 初始化随机种子
return h
}
// makemap implements Go map creation for make(map[k]v, hint).
// If the compiler has determined that the map or the first bucket
// can be created on the stack, h and/or bucket may be non-nil.
// If h != nil, the map can be created directly in h.
// If h.buckets != nil, bucket pointed to can be used as the first bucket.
// makemap 实现 make(map[k]v, hint)
func makemap(t *maptype, hint int, h *hmap) *hmap {
// 计算内存大小 hint * t.bucket.size
mem, overflow := math.MulUintptr(uintptr(hint), t.bucket.size)
if overflow || mem > maxAlloc { // 溢出或超过最大内存限制
hint = 0
}
// initialize Hmap
// 初始化 hmap
if h == nil {
h = new(hmap)
}
h.hash0 = fastrand() // 初始化随机种子
// Find the size parameter B which will hold the requested # of elements.
// For hint < 0 overLoadFactor returns false since hint < bucketCnt.
// 计算 B
B := uint8(0)
for overLoadFactor(hint, B) {
B++
}
h.B = B
// allocate initial hash table
// if B == 0, the buckets field is allocated lazily later (in mapassign)
// If hint is large zeroing this memory could take a while.
// 分配初始化哈希表。B=0的时候,在 mapassign 延迟分配
if h.B != 0 { // B != 0 时初始化桶指针buckets
var nextOverflow *bmap
// 分配 buckets
h.buckets, nextOverflow = makeBucketArray(t, h.B, nil)
if nextOverflow != nil {
// 如果有多余的,设置到 nextOverflow
h.extra = new(mapextra)
h.extra.nextOverflow = nextOverflow
}
}
return h
}
// makeBucketArray initializes a backing array for map buckets.
// 1<<b is the minimum number of buckets to allocate.
// dirtyalloc should either be nil or a bucket array previously
// allocated by makeBucketArray with the same t and b parameters.
// If dirtyalloc is nil a new backing array will be alloced and
// otherwise dirtyalloc will be cleared and reused as backing array.
// makeBucketArray 初始化哈希桶
// 1<<b 是最小分配数目
// 参数 dirtyalloc 要么为空,要么是之前使用相同的 t 和 b 参数调用 makeBucketArray 分配的哈希桶数组。
// dirtyalloc 为 nil ,生产新的; 否则会清除作为
func makeBucketArray(t *maptype, b uint8, dirtyalloc unsafe.Pointer) (buckets unsafe.Pointer, nextOverflow *bmap) {
// base 代表用户预期的桶的数量,即hash数组的真实大小,2^B
base := bucketShift(b)
nbuckets := base
// For small b, overflow buckets are unlikely.
// Avoid the overhead of the calculation.
// 对于小 b 不太可能有溢出桶,避免计算开销。 也就是 base <= 8 的时候
if b >= 4 {
// Add on the estimated number of overflow buckets
// required to insert the median number of elements
// used with this value of b.
// 加上一些预计的溢出桶数。
nbuckets += bucketShift(b - 4)
// 计算需要的内存
sz := t.bucket.size * nbuckets
// 计算 mallocgc 分配的实际大小
up := roundupsize(sz)
if up != sz {
// 重新计算 nbuckets
nbuckets = up / t.bucket.size
}
}
if dirtyalloc == nil {
buckets = newarray(t.bucket, int(nbuckets))
} else {
// dirtyalloc was previously generated by
// the above newarray(t.bucket, int(nbuckets))
// but may not be empty.
// dirtyalloc 由上面 newarray 生产的,可能不为空
buckets = dirtyalloc
size := t.bucket.size * nbuckets
// 根据是由有指针,清空下
if t.bucket.kind&kindNoPointers == 0 {
memclrHasPointers(buckets, size)
} else {
memclrNoHeapPointers(buckets, size)
}
}
// 如果 nbuckets 不等于 base, 则把多余的放到 nextOverflow 中
if base != nbuckets {
// We preallocated some overflow buckets.
// To keep the overhead of tracking these overflow buckets to a minimum,
// we use the convention that if a preallocated overflow bucket's overflow
// pointer is nil, then there are more available by bumping the pointer.
// We need a safe non-nil pointer for the last overflow bucket; just use buckets.
// 预分配一些溢出桶
// 为了最小化跟踪溢出桶的开销,规定:如果溢出桶的 overflow 指针为空,则后面还有更多的溢出桶,
// 在最后设置 overflow 为非空的指针,直接用 buckets 就好了。
nextOverflow = (*bmap)(add(buckets, base*uintptr(t.bucketsize)))
last := (*bmap)(add(buckets, (nbuckets-1)*uintptr(t.bucketsize)))
last.setoverflow(t, (*bmap)(buckets))
}
return buckets, nextOverflow
}
// mapaccess1 returns a pointer to h[key]. Never returns nil, instead
// it will return a reference to the zero object for the value type if
// the key is not in the map.
// NOTE: The returned pointer may keep the whole map live, so don't
// hold onto it for very long.
// mapaccess1 返回 h[key] 。当 key 不存在 map 中,永远不会返回 nil ,而是返回零值(zeroVal)的引用。
// 注意:返回的值不要一直持有,这样会让 map 一直存活。
// mapaccess1 为 v := m[k] 的时候调用,只返回值
func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
if raceenabled && h != nil {
callerpc := getcallerpc()
pc := funcPC(mapaccess1)
racereadpc(unsafe.Pointer(h), callerpc, pc)
raceReadObjectPC(t.key, key, callerpc, pc)
}
if msanenabled && h != nil {
msanread(key, t.key.size)
}
// h 为空,返回 zeroVal
if h == nil || h.count == 0 {
if t.hashMightPanic() {
t.key.alg.hash(key, 0) // see issue 23734
}
return unsafe.Pointer(&zeroVal[0])
}
// 如果处于写状态,panic
if h.flags&hashWriting != 0 {
throw("concurrent map read and map write")
}
alg := t.key.alg
hash := alg.hash(key, uintptr(h.hash0)) // 计算哈希值
m := bucketMask(h.B) // 获取掩码
b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize))) // 通过低 hash&m (也就是低 B 位) 来计算处于哪一个槽,得到哈希桶
if c := h.oldbuckets; c != nil {
// 在扩容,判断是否为相等大小扩容
if !h.sameSizeGrow() {
// There used to be half as many buckets; mask down one more power of two.
// 这里是翻倍扩容,B 已经+1了,所以需要 m>>1
m >>= 1
}
// 计算之前的哈希桶
oldb := (*bmap)(add(c, (hash&m)*uintptr(t.bucketsize)))
// 如果还没有迁移过,则 b = oldb
if !evacuated(oldb) {
b = oldb
}
}
// 获取高 8 位
top := tophash(hash)
bucketloop:
// 遍历哈希桶和对应的溢出桶(也是哈希桶),来查找
for ; b != nil; b = b.overflow(t) {
// 遍历哈希桶 8 个位置
for i := uintptr(0); i < bucketCnt; i++ {
// 存高 8 位,是为了快速判断,过滤不满足的
if b.tophash[i] != top {
if b.tophash[i] == emptyRest {
// emptyRest 表示高索引和后面的溢出桶都没有非空的元素了,直接跳出循环,未找到
break bucketloop
}
continue
}
// 此时 b.tophash[i] == top ,再判断 key 值是否相等
k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
if t.indirectkey() {
// k 是指针类型,解指针,获取对应的值
k = *((*unsafe.Pointer)(k))
}
// 判断是否相等
if alg.equal(key, k) {
// 找到了,返回 value
v := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize))
if t.indirectvalue() {
// v 是指针类型,解指针,获取对应的值
v = *((*unsafe.Pointer)(v))
}
return v
}
}
}
// 未找到
return unsafe.Pointer(&zeroVal[0])
}
// mapaccess2 为 v,ok := m[k] 的时候调用,返回值和是否找到。 和 mapaccess1 实现几乎一致。
func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool) {
if raceenabled && h != nil {
callerpc := getcallerpc()
pc := funcPC(mapaccess2)
racereadpc(unsafe.Pointer(h), callerpc, pc)
raceReadObjectPC(t.key, key, callerpc, pc)
}
if msanenabled && h != nil {
msanread(key, t.key.size)
}
if h == nil || h.count == 0 {
if t.hashMightPanic() {
t.key.alg.hash(key, 0) // see issue 23734
}
return unsafe.Pointer(&zeroVal[0]), false
}
if h.flags&hashWriting != 0 {
throw("concurrent map read and map write")
}
alg := t.key.alg
hash := alg.hash(key, uintptr(h.hash0))
m := bucketMask(h.B)
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
// There used to be half as many buckets; mask down one more power of two.
m >>= 1
}
oldb := (*bmap)(unsafe.Pointer(uintptr(c) + (hash&m)*uintptr(t.bucketsize)))
if !evacuated(oldb) {
b = oldb
}
}
top := tophash(hash)
bucketloop:
for ; b != nil; b = b.overflow(t) {
for i := uintptr(0); i < bucketCnt; i++ {
if b.tophash[i] != top {
if b.tophash[i] == emptyRest {
break bucketloop
}
continue
}
k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
if t.indirectkey() {
k = *((*unsafe.Pointer)(k))
}
if alg.equal(key, k) {
v := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize))
if t.indirectvalue() {
v = *((*unsafe.Pointer)(v))
}
return v, true
}
}
}
return unsafe.Pointer(&zeroVal[0]), false
}
// returns both key and value. Used by map iterator
// mapaccessK 返回 key 和 value ,迭代器中使用
func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer) {
if h == nil || h.count == 0 {
return nil, nil
}
alg := t.key.alg
hash := alg.hash(key, uintptr(h.hash0))
m := bucketMask(h.B)
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
// There used to be half as many buckets; mask down one more power of two.
m >>= 1
}
oldb := (*bmap)(unsafe.Pointer(uintptr(c) + (hash&m)*uintptr(t.bucketsize)))
if !evacuated(oldb) {
b = oldb
}
}
top := tophash(hash)
bucketloop:
for ; b != nil; b = b.overflow(t) {
for i := uintptr(0); i < bucketCnt; i++ {
if b.tophash[i] != top {
if b.tophash[i] == emptyRest {
break bucketloop
}
continue
}
k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
if t.indirectkey() {
k = *((*unsafe.Pointer)(k))
}
if alg.equal(key, k) {
v := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize))
if t.indirectvalue() {
v = *((*unsafe.Pointer)(v))
}
return k, v
}
}
}
return nil, nil
}
// cmd/compile/internal/gc/walk.go 中可以看到,当 val > 1024(maxZero,也就是 sizeof(zeroVal))的时候调用这个
// 否则调用 mapaccess1 ,> 1024 就不能引用 maxZero 了
func mapaccess1_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) unsafe.Pointer {
v := mapaccess1(t, h, key)
if v == unsafe.Pointer(&zeroVal[0]) {
return zero
}
return v
}
func mapaccess2_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) (unsafe.Pointer, bool) {
v := mapaccess1(t, h, key)
if v == unsafe.Pointer(&zeroVal[0]) {
return zero, false
}
return v, true
}
// Like mapaccess, but allocates a slot for the key if it is not present in the map.
// mapassign 类似于 mapaccess ,但是当没有 key ,也分配对应的单元
// mapassign 赋值: 实现是查找一个空的bucket,把key赋值到bucket上,然后把val的地址返回,然后直接通过汇编做内存拷贝
func mapassign(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
if h == nil {
panic(plainError("assignment to entry in nil map"))
}
if raceenabled {
callerpc := getcallerpc()
pc := funcPC(mapassign)
racewritepc(unsafe.Pointer(h), callerpc, pc)
raceReadObjectPC(t.key, key, callerpc, pc)
}
if msanenabled {
msanread(key, t.key.size)
}
// 并发写
if h.flags&hashWriting != 0 {
throw("concurrent map writes")
}
alg := t.key.alg
hash := alg.hash(key, uintptr(h.hash0))
// Set hashWriting after calling alg.hash, since alg.hash may panic,
// in which case we have not actually done a write.
// 调用 alg.hash 后再设置 hashWriting ,因为 alg.hash 可能 panic ,在这种情况下,我们还没有真正的写数据
h.flags ^= hashWriting
if h.buckets == nil {
h.buckets = newobject(t.bucket) // newarray(t.bucket, 1)
}
again:
// 计算 hash 对应的哈希桶在 map 中的的位置
bucket := hash & bucketMask(h.B)
// 如果在迁移
if h.growing() {
// 确保 bucket 对应的 oldbucket 已经迁移完了
growWork(t, h, bucket)
}
// 获取哈希桶
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize)))
top := tophash(hash) // 获取高 8 位,用于快速过来不满足的
var inserti *uint8 // 插入的索引位置
var insertk unsafe.Pointer // 插入的 key 的位置
var val unsafe.Pointer // 插入的 val 的位置
bucketloop:
// 遍历哈希桶和对应的溢出桶(也是哈希桶),来查找
for {
// 遍历哈希桶 8 个位置
for i := uintptr(0); i < bucketCnt; i++ {
// 存高 8 位,是为了快速判断,过滤不满足的
if b.tophash[i] != top {
// 如果为 nil ,则记录插入位置
if isEmpty(b.tophash[i]) && inserti == nil {
inserti = &b.tophash[i]
insertk = add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize))
}
if b.tophash[i] == emptyRest {
// emptyRest 表示高索引和后面的溢出桶都没有非空的元素了,直接跳出循环,未找到
break bucketloop
}
continue
}
// 此时 b.tophash[i] == top ,再判断 key 值是否相等
k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
if t.indirectkey() {
// k 是指针类型,解指针,获取对应的值
k = *((*unsafe.Pointer)(k))
}
// 判断是否相等,不相等则继续找
if !alg.equal(key, k) {
continue
}
// 此处 key == k
// already have a mapping for key. Update it.
// 已经有对应的映射, 判断 key 是否要更新,如果需要则更新下
// 可以参考:src/cmd/compile/internal/gc/reflect.go
// 当key类型为float32\float64\complex64\complex64\interface\string,或字段里面包含这些类型的都需要更新key。
if t.needkeyupdate() {
typedmemmove(t.key, k, key)
}
// 获取插入 val 的指针
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize))
// 找到了之前的值
goto done
}
// 迭代溢出桶
ovf := b.overflow(t)
if ovf == nil {
break
}
b = ovf
}
// Did not find mapping for key. Allocate new cell & add entry.
// 没有找到 key 的映射,分配并添加
// If we hit the max load factor or we have too many overflow buckets,
// and we're not already in the middle of growing, start growing.
// 如果达到了负载因子或者有太多的溢出桶,并且不在扩容的时候,那么开始扩容吧
if !h.growing() && (overLoadFactor(h.count+1, h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) {
hashGrow(t, h)
goto again // Growing the table invalidates everything, so try again // 扩容的过程导致之前的无效,再次尝试
}
// 如果没有找到插入位置,则新建一个溢出桶,并记录插入位置为新溢出桶的第一个位置
if inserti == nil {
// all current buckets are full, allocate a new one.
newb := h.newoverflow(t, b)
inserti = &newb.tophash[0]
insertk = add(unsafe.Pointer(newb), dataOffset)
val = add(insertk, bucketCnt*uintptr(t.keysize))
}
// store new key/value at insert position
if t.indirectkey() {
// key 是指针类型,设置对应的零指针
kmem := newobject(t.key)
*(*unsafe.Pointer)(insertk) = kmem
insertk = kmem
}
if t.indirectvalue() {
// value 是指针类型,设置对应的零指针
vmem := newobject(t.elem)
*(*unsafe.Pointer)(val) = vmem
}
// 设置 key, tophash, count
typedmemmove(t.key, insertk, key)
*inserti = top
h.count++
done:
// 并发写
if h.flags&hashWriting == 0 {
throw("concurrent map writes")
}
// &^ 是 bit clear ,清除 flags 中hashWriting 中为 1 的位,也就是清除 hashWriting 标记
h.flags &^= hashWriting
if t.indirectvalue() {
// val 是指针类型,解指针,获取对应的值
val = *((*unsafe.Pointer)(val))
}
// 返回的是 value 对应的指针,汇编会负责去填充
return val
}
// map 删除
func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
if raceenabled && h != nil {
callerpc := getcallerpc()
pc := funcPC(mapdelete)
racewritepc(unsafe.Pointer(h), callerpc, pc)
raceReadObjectPC(t.key, key, callerpc, pc)
}
if msanenabled && h != nil {
msanread(key, t.key.size)
}
// map 为空,直接返回
if h == nil || h.count == 0 {
if t.hashMightPanic() {
t.key.alg.hash(key, 0) // see issue 23734
}
return
}
// 并发写
if h.flags&hashWriting != 0 {
throw("concurrent map writes")
}
alg := t.key.alg
hash := alg.hash(key, uintptr(h.hash0))
// Set hashWriting after calling alg.hash, since alg.hash may panic,
// in which case we have not actually done a write (delete).
// 调用 alg.hash 后再设置 hashWriting ,因为 alg.hash 可能 panic ,在这种情况下,我们还没有真正的写数据
h.flags ^= hashWriting
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
b := (*bmap)(add(h.buckets, bucket*uintptr(t.bucketsize)))
bOrig := b
top := tophash(hash)
search:
// 遍历哈希桶和对应的溢出桶(也是哈希桶),来查找
for ; b != nil; b = b.overflow(t) {
// 遍历哈希桶 8 个位置
for i := uintptr(0); i < bucketCnt; i++ {
// 存高 8 位,是为了快速判断,过滤不满足的
if b.tophash[i] != top {
// emptyRest 表示高索引和后面的溢出桶都没有非空的元素了,直接跳出循环,未找到
if b.tophash[i] == emptyRest {
break search
}
continue
}
// 此时 b.tophash[i] == top ,再判断 key 值是否相等
k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
k2 := k
if t.indirectkey() {
// k 是指针类型,解指针,获取对应的值
k2 = *((*unsafe.Pointer)(k2))
}
// 判断是否相等,不相等则继续找
if !alg.equal(key, k2) {
continue
}
// Only clear key if there are pointers in it.
// 只有当 key 包含指针的时候才清除
if t.indirectkey() {
// key 是指针类型,设置对应的指针为 nil 即可
*(*unsafe.Pointer)(k) = nil
} else if t.key.kind&kindNoPointers == 0 {
// key 是包含指针的类型, 调用 memclrHasPointers 来清理
memclrHasPointers(k, t.key.size)
}
// 获取 value 对应的地址,并清理
v := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize))
if t.indirectvalue() {
// value 是指针类型,设置对应的指针为 nil 即可
*(*unsafe.Pointer)(v) = nil
} else if t.elem.kind&kindNoPointers == 0 {
// value 是包含指针的类型, 调用 memclrHasPointers 来清理
memclrHasPointers(v, t.elem.size)
} else {
// 否则调用 memclrNoHeapPointers 来清理
memclrNoHeapPointers(v, t.elem.size)
}
// 设置 emptyOne 标记
b.tophash[i] = emptyOne
// If the bucket now ends in a bunch of emptyOne states,
// change those to emptyRest states.
// It would be nice to make this a separate function, but
// for loops are not currently inlineable.
// 如果是是以一堆 emptyOne 状态结束,修改成 emptyRest 状态
if i == bucketCnt-1 {
// 后面还有溢出桶 && 溢出桶中还有非空的
if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest {
goto notLast // 不是最后一个非空的
}
} else {
// 后面还有非空的
if b.tophash[i+1] != emptyRest {
goto notLast // 不是最后一个非空的
}
}
// 表示 b.tophash[i] 后面都是空的
// 然后,又将此值删除了,那么往前走,将满足的都设置为 emptyRest
for {
b.tophash[i] = emptyRest
// i == 0,找前面一个哈希桶
if i == 0 {
// 找到头了
if b == bOrig {
break // beginning of initial bucket, we're done.
}
// Find previous bucket, continue at its last entry.
// 找前一个哈希桶
c := b
for b = bOrig; b.overflow(t) != c; b = b.overflow(t) {
}
i = bucketCnt - 1
} else {
// 哈希桶内,往前移一格
i--
}
// 找到不为 emptyOne 的时候,跳出循环
if b.tophash[i] != emptyOne {
break
}
}
notLast:
// 数量减 1 ,再跳出循环
h.count--
break search
}
}
// 并发写
if h.flags&hashWriting == 0 {
throw("concurrent map writes")
}
// &^ 是 bit clear ,清除 flags 中 hashWriting 中为 1 的位,也就是清除 hashWriting 标记
h.flags &^= hashWriting
}
// mapiterinit initializes the hiter struct used for ranging over maps.
// The hiter struct pointed to by 'it' is allocated on the stack
// by the compilers order pass or on the heap by reflect_mapiterinit.
// Both need to have zeroed hiter since the struct contains pointers.
// mapiterinit 初始化 hiter 结构体,用来 for :=range m 使用。
// 指向 hiter 的 it 通过编译器在栈上分配,或者通过 reflect_mapiterinit 在堆上分配,都需要清零。
func mapiterinit(t *maptype, h *hmap, it *hiter) {
if raceenabled && h != nil {
callerpc := getcallerpc()
racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapiterinit))
}
// map 为空,返回
if h == nil || h.count == 0 {
return
}
if unsafe.Sizeof(hiter{})/sys.PtrSize != 12 {
throw("hash_iter size incorrect") // see cmd/compile/internal/gc/reflect.go
}
it.t = t
it.h = h
// grab snapshot of bucket state
// 创建当前 map 状态快照
it.B = h.B
it.buckets = h.buckets
if t.bucket.kind&kindNoPointers != 0 {
// Allocate the current slice and remember pointers to both current and old.
// This preserves all relevant overflow buckets alive even if
// the table grows and/or overflow buckets are added to the table
// while we are iterating.
// 分配当前切片并将指针记住到当前和老的。
// 这将使得所有相关的溢出桶 alive ,即使在迭代的时候 map 扩容、添加溢出桶。
h.createOverflow()
it.overflow = h.extra.overflow
it.oldoverflow = h.extra.oldoverflow
}