-
Notifications
You must be signed in to change notification settings - Fork 1.2k
/
elasticsearch_transport.go
111 lines (96 loc) · 2.43 KB
/
elasticsearch_transport.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
package tracer
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"time"
"github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v7/esutil"
)
const (
ElasticSearchDefaultIndex = "lotus-pubsub"
flushInterval = 10 * time.Second
flushBytes = 1024 * 1024 // MB
esWorkers = 2 // TODO: hardcoded
)
func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) {
conUrl, err := url.Parse(connectionString)
if err != nil {
return nil, err
}
username := conUrl.User.Username()
password, _ := conUrl.User.Password()
cfg := elasticsearch.Config{
Addresses: []string{
conUrl.Scheme + "://" + conUrl.Host,
},
Username: username,
Password: password,
Transport: &http.Transport{},
}
es, err := elasticsearch.NewClient(cfg)
if err != nil {
return nil, err
}
var esIndex string
if elasticsearchIndex != "" {
esIndex = elasticsearchIndex
} else {
esIndex = ElasticSearchDefaultIndex
}
// Create the BulkIndexer to batch ES trace submission
bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
Index: esIndex,
Client: es,
NumWorkers: esWorkers,
FlushBytes: int(flushBytes),
FlushInterval: flushInterval,
OnError: func(ctx context.Context, err error) {
log.Errorf("Error persisting queries %s", err.Error())
},
})
if err != nil {
return nil, err
}
return &elasticSearchTransport{
cl: es,
bi: bi,
esIndex: esIndex,
}, nil
}
type elasticSearchTransport struct {
cl *elasticsearch.Client
bi esutil.BulkIndexer
esIndex string
}
func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
var e interface{}
if evt.lotusTraceEvent != nil {
e = *evt.lotusTraceEvent
} else if evt.pubsubTraceEvent != nil {
e = *evt.pubsubTraceEvent
} else {
return nil
}
jsonEvt, err := json.Marshal(e)
if err != nil {
return fmt.Errorf("error while marshaling event: %s", err)
}
return est.bi.Add(
context.Background(),
esutil.BulkIndexerItem{
Action: "index",
Body: bytes.NewReader(jsonEvt),
OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) {
if err != nil {
log.Errorf("unable to submit trace - %s", err)
} else {
log.Errorf("unable to submit trace %s: %s", res.Error.Type, res.Error.Reason)
}
},
},
)
}