-
Notifications
You must be signed in to change notification settings - Fork 7
/
producer.go
110 lines (91 loc) · 2.83 KB
/
producer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
package databus
import (
"fmt"
"time"
"errors"
"github.com/Shopify/sarama"
)
type ProducerEvent interface {
//发送消息接口
SendMessage(data []byte, key string) error
//关闭生产者
Close()
}
type producerEvent struct {
address []string
topic string
isack bool
producer sarama.AsyncProducer
}
type ProducerParam struct {
Address []string
Topic string
IsAck bool
KafkaVer string
}
func NewAsyncProducer(param ProducerParam) (ProducerEvent, error) {
config := sarama.NewConfig()
config.Net.MaxOpenRequests = 10
config.Net.DialTimeout = 30 * time.Second
config.Net.ReadTimeout = 30 * time.Second
config.Net.WriteTimeout = 30 * time.Second
//等待服务器所有副本都保存成功后的响应
config.Producer.RequiredAcks = sarama.WaitForAll
//通过msg中的key生成hash值,选择分区
config.Producer.Partitioner = sarama.NewHashPartitioner
//是否等待成功和失败后的响应,只有上面的RequireAcks设置不是NoReponse这里才有用.
if param.IsAck == true {
config.Producer.Return.Successes = true
config.Producer.Return.Errors = true
}
config.Producer.Timeout = 10 * time.Second
//重试次数
config.Producer.Retry.Max = 3
config.Producer.Retry.Backoff = 100 * time.Millisecond
//设置使用的kafka版本,如果低于V0_10_0_0版本,消息中的timestrap没有作用.需要消费和生产同时配置
//注意,版本设置不对的话,kafka会返回很奇怪的错误,并且无法成功发送消息
version, err := sarama.ParseKafkaVersion(param.KafkaVer)
if err != nil {
fmt.Println("Error parsing Kafka version: %v", err)
return nil, err
}
config.Version = version
producer, err := sarama.NewAsyncProducer(param.Address, config)
if err != nil {
fmt.Println(err)
return nil, err
}
return &producerEvent{
address: param.Address,
topic: param.Topic,
isack: param.IsAck,
producer: producer,
}, nil
}
func (handle *producerEvent) SendMessage(data []byte, key string) error {
// 注意:这里的msg必须得是新构建的变量,不然你会发现发送过去的消息内容都是一样的,因为批次发送消息的关系。
msg := &sarama.ProducerMessage{
Topic: handle.topic,
Key:sarama.ByteEncoder(key),
Value:sarama.ByteEncoder(data),
}
//使用通道发送
handle.producer.Input() <- msg
if handle.isack == true {
select {
case suc := <-handle.producer.Successes():
fmt.Println("offset: ", suc.Offset, "timestamp: ", suc.Timestamp.String(), "partitions: ", suc.Partition)
return nil
case fail := <-handle.producer.Errors():
fmt.Println("err: ", fail.Err)
return fail.Err
case timeout := <-time.After(time.Second*10):
fmt.Println("ack msg error %p.", timeout)
return errors.New("ack msg timeout.")
}
}
return nil
}
func (handle *producerEvent) Close() {
handle.producer.AsyncClose()
}