Skip to content

Commit

Permalink
parallellize deduplicate filter for resolutions
Browse files Browse the repository at this point in the history
  • Loading branch information
mauliksoneji authored and khyatisoneji committed Feb 13, 2020
1 parent 262eb64 commit ff30c41
Showing 1 changed file with 16 additions and 0 deletions.
16 changes: 16 additions & 0 deletions pkg/block/fetcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,22 @@ func NewDeduplicateFilter() *DeduplicateFilter {
// TODO(bwplotka): change arg to have sorted meta instead?
// TODO(bwplotka): Explore what we can improve here - it's bit slow.
func (f *DeduplicateFilter) Filter(metas map[ulid.ULID]*metadata.Meta, synced GaugeLabeled, _ bool) {
var wg sync.WaitGroup

for _, res := range []int64{
int64(0), int64(5 * 60 * 1000), int64(60 * 60 * 1000),
} {
wg.Add(1)
go func(res int64) {
defer wg.Done()
f.filter(metas, res, synced)
}(res)
}

wg.Wait()
}

func (f *DeduplicateFilter) filter(metas map[ulid.ULID]*metadata.Meta, resolution int64, synced GaugeLabeled) {
if cap(f.metaSlice) < len(metas) {
f.metaSlice = make([]*metadata.Meta, 0, len(metas))
}
Expand Down

0 comments on commit ff30c41

Please sign in to comment.