forked from grafana/loki
-
Notifications
You must be signed in to change notification settings - Fork 1
/
util.go
96 lines (84 loc) · 2.5 KB
/
util.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
package util
import (
"context"
"fmt"
"io"
"os"
ot "github.com/opentracing/opentracing-go"
"github.com/grafana/loki/pkg/storage/stores/series/index"
"github.com/grafana/loki/pkg/util/math"
)
// DoSingleQuery is the interface for indexes that don't support batching yet.
type DoSingleQuery func(context.Context, index.Query, index.QueryPagesCallback) error
// QueryParallelism is the maximum number of subqueries run in
// parallel per higher-level query
var QueryParallelism = 100
// DoParallelQueries translates between our interface for query batching,
// and indexes that don't yet support batching.
func DoParallelQueries(
ctx context.Context, doSingleQuery DoSingleQuery, queries []index.Query,
callback index.QueryPagesCallback,
) error {
if len(queries) == 1 {
return doSingleQuery(ctx, queries[0], callback)
}
queue := make(chan index.Query)
incomingErrors := make(chan error)
n := math.Min(len(queries), QueryParallelism)
// Run n parallel goroutines fetching queries from the queue
for i := 0; i < n; i++ {
go func() {
sp, ctx := ot.StartSpanFromContext(ctx, "DoParallelQueries-worker")
defer sp.Finish()
for {
query, ok := <-queue
if !ok {
return
}
incomingErrors <- doSingleQuery(ctx, query, callback)
}
}()
}
// Send all the queries into the queue
go func() {
for _, query := range queries {
queue <- query
}
close(queue)
}()
// Now receive all the results.
var lastErr error
for i := 0; i < len(queries); i++ {
err := <-incomingErrors
if err != nil {
lastErr = err
}
}
return lastErr
}
// EnsureDirectory makes sure directory is there, if not creates it if not
func EnsureDirectory(dir string) error {
info, err := os.Stat(dir)
if os.IsNotExist(err) {
return os.MkdirAll(dir, 0o777)
} else if err == nil && !info.IsDir() {
return fmt.Errorf("not a directory: %s", dir)
}
return err
}
// ReadCloserWithContextCancelFunc helps with cancelling the context when closing a ReadCloser.
// NOTE: The consumer of ReadCloserWithContextCancelFunc should always call the Close method when it is done reading which otherwise could cause a resource leak.
type ReadCloserWithContextCancelFunc struct {
io.ReadCloser
cancel context.CancelFunc
}
func NewReadCloserWithContextCancelFunc(readCloser io.ReadCloser, cancel context.CancelFunc) io.ReadCloser {
return ReadCloserWithContextCancelFunc{
ReadCloser: readCloser,
cancel: cancel,
}
}
func (r ReadCloserWithContextCancelFunc) Close() error {
defer r.cancel()
return r.ReadCloser.Close()
}