/
hashedfanout.go
81 lines (63 loc) · 1.64 KB
/
hashedfanout.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
package worker
import (
"hash/maphash"
"sync"
)
// HashFunc converts an input to a uint value. It must be deterministic. See examples below.
type HashFunc[I any] func(I) uint
// HashedFanOut is a special purpose fan out that hashes the input so that the same keys are always processed
// by the same worker. This is used to ensure guard against race conditions in non-reentrant processors.
type HashedFanOut[I any] struct {
workerCount uint
inputs []chan I
hasher HashFunc[I]
}
func NewHashedFanOut[I any](workerCount, bufferSize uint, hasher HashFunc[I]) *HashedFanOut[I] {
inputs := make([]chan I, workerCount)
for i := uint(0); i < workerCount; i++ {
inputs[i] = make(chan I, bufferSize)
}
return &HashedFanOut[I]{
workerCount: workerCount,
inputs: inputs,
hasher: hasher,
}
}
func (f *HashedFanOut[I]) Invoke(input I) {
hash := f.hasher(input)
f.inputs[hash%f.workerCount] <- input
}
func (f *HashedFanOut[I]) Close() {
for _, c := range f.inputs {
close(c)
}
}
func (f *HashedFanOut[I]) Process(p ProcessorFunc[I]) {
wg := sync.WaitGroup{}
for _, inputChan := range f.inputs {
wg.Add(1)
go func(c chan I) {
for i := range c {
p(i)
}
wg.Done()
}(inputChan)
}
wg.Wait()
}
type KeyFunc[I, K any] func(I) K
// StringHasher given a KeyFunc that returns a string for a given input, returns a consistent hash for the string.
func StringHasher[I any](keyFunc KeyFunc[I, string]) HashFunc[I] {
var (
hash maphash.Hash
m sync.Mutex
)
return func(input I) uint {
k := keyFunc(input)
m.Lock()
defer m.Unlock()
hash.Reset()
_, _ = hash.WriteString(k)
return uint(hash.Sum64())
}
}