/
producer.go
125 lines (103 loc) · 2.76 KB
/
producer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
package kafka
import (
"context"
"errors"
"time"
"github.com/loadimpact/k6/lib"
"github.com/loadimpact/k6/stats"
kafkago "github.com/segmentio/kafka-go"
)
func (*Kafka) Writer(brokers []string, topic string) *kafkago.Writer {
return kafkago.NewWriter(kafkago.WriterConfig{
Brokers: brokers,
Topic: topic,
Balancer: &kafkago.LeastBytes{},
BatchSize: 1,
})
}
func (*Kafka) Produce(
ctx context.Context, writer *kafkago.Writer, messages []map[string]string,
keySchema string, valueSchema string) error {
state := lib.GetState(ctx)
err := errors.New("State is nil")
if state == nil {
ReportError(err, "Cannot determine state")
return err
}
kafkaMessages := make([]kafkago.Message, len(messages))
for i, message := range messages {
key := []byte(message["key"])
if keySchema != "" {
key = ToAvro(message["value"], keySchema)
}
value := []byte(message["value"])
if valueSchema != "" {
value = ToAvro(message["value"], valueSchema)
}
kafkaMessages[i] = kafkago.Message{
Key: key,
Value: value,
}
}
err = writer.WriteMessages(ctx, kafkaMessages...)
if err == ctx.Err() {
// context is cancellled, so stop
ReportWriterStats(ctx, writer.Stats())
return nil
}
if err != nil {
ReportError(err, "Failed to write message")
ReportWriterStats(ctx, writer.Stats())
return err
}
return nil
}
func ReportWriterStats(ctx context.Context, currentStats kafkago.WriterStats) error {
state := lib.GetState(ctx)
err := errors.New("State is nil")
if state == nil {
ReportError(err, "Cannot determine state")
return err
}
tags := make(map[string]string)
tags["clientid"] = currentStats.ClientID
tags["topic"] = currentStats.Topic
now := time.Now()
stats.PushIfNotDone(ctx, state.Samples, stats.Sample{
Time: now,
Metric: WriterDials,
Tags: stats.IntoSampleTags(&tags),
Value: float64(currentStats.Dials),
})
stats.PushIfNotDone(ctx, state.Samples, stats.Sample{
Time: now,
Metric: WriterWrites,
Tags: stats.IntoSampleTags(&tags),
Value: float64(currentStats.Writes),
})
stats.PushIfNotDone(ctx, state.Samples, stats.Sample{
Time: now,
Metric: WriterMessages,
Tags: stats.IntoSampleTags(&tags),
Value: float64(currentStats.Messages),
})
stats.PushIfNotDone(ctx, state.Samples, stats.Sample{
Time: now,
Metric: WriterBytes,
Tags: stats.IntoSampleTags(&tags),
Value: float64(currentStats.Bytes),
})
stats.PushIfNotDone(ctx, state.Samples, stats.Sample{
Time: now,
Metric: WriterRebalances,
Tags: stats.IntoSampleTags(&tags),
Value: float64(currentStats.Rebalances),
})
stats.PushIfNotDone(ctx, state.Samples, stats.Sample{
Time: now,
Metric: WriterErrors,
Tags: stats.IntoSampleTags(&tags),
Value: float64(currentStats.Errors),
})
return nil
}