forked from olivere/elastic
-
Notifications
You must be signed in to change notification settings - Fork 0
/
tracing.go
193 lines (170 loc) · 4.64 KB
/
tracing.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.
// Tracing is the same as the bulk_insert recipe, but adds
// OpenTracing support.
//
// Example
//
// Bulk index 100.000 documents into the index "warehouse", type "product",
// committing every set of 1.000 documents.
//
// ./run-tracer.sh
// ./tracing -index=warehouse -type=product -n=100000 -bulk-size=1000
//
package main
import (
"context"
"encoding/base64"
"errors"
"flag"
"fmt"
"log"
"math/rand"
"net/http"
"sync/atomic"
"time"
jaegerconfig "github.com/uber/jaeger-client-go/config"
jaegerlog "github.com/uber/jaeger-client-go/log"
jaegerexpvar "github.com/uber/jaeger-lib/metrics/expvar"
"golang.org/x/sync/errgroup"
"github.com/olivere/elastic"
"github.com/olivere/elastic/trace/opentracing"
)
func main() {
var (
url = flag.String("url", "http://localhost:9200", "Elasticsearch URL")
index = flag.String("index", "", "Elasticsearch index name")
typ = flag.String("type", "", "Elasticsearch type name")
sniff = flag.Bool("sniff", true, "Enable or disable sniffing")
n = flag.Int("n", 0, "Number of documents to bulk insert")
bulkSize = flag.Int("bulk-size", 0, "Number of documents to collect before committing")
)
flag.Parse()
log.SetFlags(0)
rand.Seed(time.Now().UnixNano())
if *url == "" {
log.Fatal("missing url parameter")
}
if *index == "" {
log.Fatal("missing index parameter")
}
if *typ == "" {
log.Fatal("missing type parameter")
}
if *n <= 0 {
log.Fatal("n must be a positive number")
}
if *bulkSize <= 0 {
log.Fatal("bulk-size must be a positive number")
}
opts := []elastic.ClientOptionFunc{
elastic.SetURL(*url),
elastic.SetSniff(*sniff),
}
// Initialize Jaeger tracing
cfg := jaegerconfig.Configuration{}
closer, err := cfg.InitGlobalTracer(
"elastic-tracing",
jaegerconfig.Logger(jaegerlog.StdLogger),
jaegerconfig.Metrics(jaegerexpvar.NewFactory(10)),
)
if err != nil {
log.Fatalf("unable to initialize jaeger tracer: %v", err)
}
defer closer.Close()
httpClient := &http.Client{
Transport: opentracing.NewTransport(),
}
opts = append(opts, elastic.SetHttpClient(httpClient))
// Create an Elasticsearch client
client, err := elastic.NewClient(opts...)
if err != nil {
log.Fatal(err)
}
// Setup a group of goroutines from the excellent errgroup package
g, ctx := errgroup.WithContext(context.TODO())
// The first goroutine will emit documents and send it to the second goroutine
// via the docsc channel.
// The second Goroutine will simply bulk insert the documents.
type doc struct {
ID string `json:"id"`
Timestamp time.Time `json:"@timestamp"`
}
docsc := make(chan doc)
begin := time.Now()
// Goroutine to create documents
g.Go(func() error {
defer close(docsc)
buf := make([]byte, 32)
for i := 0; i < *n; i++ {
// Generate a random ID
_, err := rand.Read(buf)
if err != nil {
return err
}
id := base64.URLEncoding.EncodeToString(buf)
// Construct the document
d := doc{
ID: id,
Timestamp: time.Now(),
}
// Send over to 2nd goroutine, or cancel
select {
case docsc <- d:
case <-ctx.Done():
return ctx.Err()
}
}
return nil
})
// Second goroutine will consume the documents sent from the first and bulk insert into ES
var total uint64
g.Go(func() error {
bulk := client.Bulk().Index(*index).Type(*typ)
for d := range docsc {
// Simple progress
current := atomic.AddUint64(&total, 1)
dur := time.Since(begin).Seconds()
sec := int(dur)
pps := int64(float64(current) / dur)
fmt.Printf("%10d | %6d req/s | %02d:%02d\r", current, pps, sec/60, sec%60)
// Enqueue the document
bulk.Add(elastic.NewBulkIndexRequest().Id(d.ID).Doc(d))
if bulk.NumberOfActions() >= *bulkSize {
// Commit
res, err := bulk.Do(ctx)
if err != nil {
return err
}
if res.Errors {
// Look up the failed documents with res.Failed(), and e.g. recommit
return errors.New("bulk commit failed")
}
// "bulk" is reset after Do, so you can reuse it
}
select {
default:
case <-ctx.Done():
return ctx.Err()
}
}
// Commit the final batch before exiting
if bulk.NumberOfActions() > 0 {
_, err = bulk.Do(ctx)
if err != nil {
return err
}
}
return nil
})
// Wait until all goroutines are finished
if err := g.Wait(); err != nil {
log.Fatal(err)
}
// Final results
dur := time.Since(begin).Seconds()
sec := int(dur)
pps := int64(float64(total) / dur)
fmt.Printf("%10d | %6d req/s | %02d:%02d\n", total, pps, sec/60, sec%60)
}