/
parallel.go
60 lines (51 loc) · 1.16 KB
/
parallel.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
package parallel
import (
"context"
"github.com/michaelquigley/pfxlog"
"github.com/openziti/fablab/kernel/lib/util"
"github.com/pkg/errors"
"golang.org/x/sync/semaphore"
"sync/atomic"
)
type Task func() error
func Execute(tasks []Task, concurrency int64) error {
if concurrency < 1 {
return errors.Errorf("invalid concurrency %v, must be at least 1", concurrency)
}
completed := atomic.Int64{}
sem := semaphore.NewWeighted(concurrency)
errorsC := make(chan error, len(tasks))
for _, task := range tasks {
if err := sem.Acquire(context.Background(), 1); err != nil {
errorsC <- err
continue
}
boundTask := task
go func() {
defer func() {
sem.Release(1)
current := completed.Add(1)
if current%10 == 0 {
pfxlog.Logger().Infof("completed %d/%d tasks", current, len(tasks))
}
if int(current) == len(tasks) {
close(errorsC)
}
}()
if err := boundTask(); err != nil {
errorsC <- err
}
}()
}
var errList []error
for err := range errorsC {
errList = append(errList, err)
}
if len(errList) == 0 {
return nil
}
if len(errList) == 1 {
return errList[0]
}
return util.MultipleErrors(errList)
}