Skip to content

Commit

Permalink
feat(performance): Execute collection concurrently per pool
Browse files Browse the repository at this point in the history
This should be safe from introducing excessive load in most
configurations, since pools generally do not share disks.
  • Loading branch information
pdf committed Aug 14, 2021
1 parent 53b0e98 commit ccc6f22
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 8 deletions.
21 changes: 17 additions & 4 deletions collector/dataset.go
Expand Up @@ -2,6 +2,7 @@ package collector

import (
"fmt"
"sync"

"github.com/go-kit/log"
"github.com/go-kit/log/level"
Expand Down Expand Up @@ -170,13 +171,25 @@ type datasetCollector struct {
}

func (c *datasetCollector) update(ch chan<- metric, pools []string, excludes regexpCollection) error {
var wg sync.WaitGroup
errChan := make(chan error, len(pools))
for _, pool := range pools {
if err := c.updatePoolMetrics(ch, pool, excludes); err != nil {
return err
}
wg.Add(1)
go func(pool string) {
if err := c.updatePoolMetrics(ch, pool, excludes); err != nil {
errChan <- err
}
wg.Done()
}(pool)
}
wg.Wait()

return nil
select {
case err := <-errChan:
return err
default:
return nil
}
}

func (c *datasetCollector) updatePoolMetrics(ch chan<- metric, pool string, excludes regexpCollection) error {
Expand Down
21 changes: 17 additions & 4 deletions collector/pool.go
Expand Up @@ -2,6 +2,7 @@ package collector

import (
"fmt"
"sync"

"github.com/go-kit/log"
"github.com/go-kit/log/level"
Expand Down Expand Up @@ -116,13 +117,25 @@ type poolCollector struct {
}

func (c *poolCollector) update(ch chan<- metric, pools []string, excludes regexpCollection) error {
var wg sync.WaitGroup
errChan := make(chan error, len(pools))
for _, pool := range pools {
if err := c.updatePoolMetrics(ch, pool); err != nil {
return err
}
wg.Add(1)
go func(pool string) {
if err := c.updatePoolMetrics(ch, pool); err != nil {
errChan <- err
}
wg.Done()
}(pool)
}
wg.Wait()

return nil
select {
case err := <-errChan:
return err
default:
return nil
}
}

func (c *poolCollector) updatePoolMetrics(ch chan<- metric, pool string) error {
Expand Down

0 comments on commit ccc6f22

Please sign in to comment.