/
cache_filters.go
78 lines (70 loc) · 1.75 KB
/
cache_filters.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
package bloom
import (
"encoding/csv"
"os"
"path/filepath"
"strconv"
"github.com/devopsfaith/bloomfilter"
baseBloomfilter "github.com/devopsfaith/bloomfilter/bloomfilter"
"github.com/gnames/gnsys"
"github.com/rs/zerolog/log"
)
// filtersFromcache unmarchals data from a file, and uses the data for the
// filter creations.
func (em *exactMatcher) filtersFromCache(path string) error {
cPath := filepath.Join(path, canonicalStemFile)
sizesPath := filepath.Join(path, sizesFile)
cPathExists, err := gnsys.FileExists(cPath)
if err != nil {
return err
}
if cPathExists {
if err := em.getFiltersFromCache(cPath, sizesPath); err != nil {
return err
}
}
return nil
}
func (em *exactMatcher) getFiltersFromCache(cPath, sizesPath string) error {
log.Info().Msg("Geting bloom lookup data from a cache on disk")
cCfg := restoreConfigs(sizesPath)
cFilter := baseBloomfilter.New(cCfg)
cBytes, err := os.ReadFile(cPath)
if err != nil {
return err
}
if err = cFilter.UnmarshalBinary(cBytes); err != nil {
return err
}
em.filters = &bloomFilters{
canonicalStem: cFilter,
}
return nil
}
func restoreConfigs(sizeFile string) bloomfilter.Config {
var cCfg bloomfilter.Config
f, err := os.Open(sizeFile)
if err != nil {
log.Warn().Err(err).Msgf("Could not open file %s", sizeFile)
}
r := csv.NewReader(f)
rows, err := r.ReadAll()
if err != nil {
log.Warn().Err(err).Msgf("Could not read data from file %s", sizeFile)
}
for _, v := range rows {
size, err := strconv.Atoi(v[1])
if err != nil {
log.Warn().Err(err).Msg("Could not convert data size to int")
}
switch v[0] {
case "CanonicalSize":
cCfg = bloomfilter.Config{
N: uint(size),
P: 0.00001,
HashName: bloomfilter.HASHER_OPTIMAL,
}
}
}
return cCfg
}