-
Notifications
You must be signed in to change notification settings - Fork 2
/
kafka_0_11.go
111 lines (99 loc) · 2.62 KB
/
kafka_0_11.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
package subscribers
import (
"encoding/json"
"fmt"
"github.com/murderxchip/aliyun-dts-dispatcher/define"
"github.com/murderxchip/aliyun-dts-dispatcher/dts/consumers"
"github.com/murderxchip/aliyun-dts-dispatcher/log"
"github.com/Shopify/sarama"
"time"
)
type Kafka011Subscriber struct {
SubscriberBase
producer sarama.SyncProducer
mqConfig MqConfig
}
func (s Kafka011Subscriber) GetConfigHash() []byte {
return s.ConfigHash
}
func (s Kafka011Subscriber) GetForceUpdate() bool {
return s.ForceUpdate
}
type MqConfig struct {
Topics []string `json:"topics"`
Servers []string `json:"servers"`
ConsumerId string `json:"consumerGroup"`
}
func NewKafka011Subscriber(config SubscriberKafka011Config) *Kafka011Subscriber {
kafkaCfg := MqConfig{
Topics: config.Topics,
Servers: config.Servers,
ConsumerId: config.ConsumerId,
}
s := &Kafka011Subscriber{
SubscriberBase{
Name: config.Name,
Consumer: config.Consumer,
Queue: make(chan consumers.DtsData, 2000),
ForceUpdate: config.ForceUpdate,
ConfigHash: config.Hash(),
},
nil,
kafkaCfg,
}
s.SetTables(config.Tables)
mqCfg := sarama.NewConfig()
mqCfg.Producer.Return.Successes = true
mqCfg.Version = sarama.V0_11_0_0
if err := mqCfg.Validate(); err != nil {
msg := fmt.Sprintf("Kafka producer config invalidate. config: %v. err: %v", kafkaCfg, err)
fmt.Println(msg)
log.Fatal(define.Kafka,msg)
panic(msg)
}
producer, err := sarama.NewSyncProducer(kafkaCfg.Servers, mqCfg)
if err != nil {
msg := fmt.Sprintf("Kafak producer create fail. err: %v", err)
log.Fatal(define.Kafka,msg)
panic(msg)
}
s.producer = producer
return s
}
func (s Kafka011Subscriber) Run() {
log.Info(define.SubscribersTag, "starting subscriber routine", s.Name)
go func() {
for {
select {
case data := <-s.Queue:
//log.Info(define.SubscribersTag,"get dts message")
str, err := json.Marshal(data)
if err != nil {
log.Error(define.SubscribersTag, "get dts error", err)
} else {
msg := &sarama.ProducerMessage{
Topic: s.mqConfig.Topics[0],
Key: sarama.StringEncoder(""),
Value: sarama.StringEncoder(str),
Timestamp: time.Now(),
}
_, _, err := s.producer.SendMessage(msg)
if err != nil {
log.Error(define.SubscribersTag, "push msg error", err)
return
} else {
//successful
//log.Info(define.SubscribersTag, "Publish ok", str)
}
}
default:
if s.Quit == 1 && len(s.Queue) == 0 {
//todo quit subscriber
log.Error(define.SubscribersTag, "break sub routine")
break
}
time.Sleep(time.Second * 1)
}
}
}()
}