/
kafka.go
77 lines (64 loc) · 1.78 KB
/
kafka.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
package connectors
import (
"context"
"fmt"
"github.com/dimpogissou/isengard-server/config"
"github.com/dimpogissou/isengard-server/logger"
"github.com/hpcloud/tail"
uuid "github.com/nu7hatch/gouuid"
"github.com/segmentio/kafka-go"
)
func SetupKafkaConnection(host string, port string, topic string) *kafka.Writer {
writer := kafka.NewWriter(kafka.WriterConfig{
Brokers: []string{fmt.Sprintf("%s:%s", host, port)},
Topic: topic,
Balancer: &kafka.LeastBytes{},
})
return writer
}
func CloseKafkaConnection(writer *kafka.Writer) error {
if err := writer.Close(); err != nil {
logger.Error("FailedClosingKafkaPublisher", err.Error())
return err
}
logger.Info("Closed Kafka publisher ...")
return nil
}
func (c KafkaConnector) writeKafkaMessages(key string, message string) error {
err := c.writer.WriteMessages(context.Background(),
kafka.Message{
Key: []byte(key),
Value: []byte(message),
})
if err != nil {
return err
}
logger.Info(fmt.Sprintf("Successfully published message to Kafka at key [%s] -> %s", key, message))
return nil
}
type KafkaConnector struct {
cfg config.KafkaConnectorConfig
writer *kafka.Writer
}
func (c KafkaConnector) GetName() string {
return c.cfg.Name
}
func (c KafkaConnector) Close() error {
err := CloseKafkaConnection(c.writer)
return err
}
func (c KafkaConnector) Send(line *tail.Line) error {
logger.Debug(fmt.Sprintf("Sending line to Kafka --> %v", line.Text))
uuid, err := uuid.NewV4()
if err != nil {
logger.Error("CreateUuidError", err.Error())
return err
}
// TODO -> Optimize string write since this operation is repeated for each log line
err = c.writeKafkaMessages(fmt.Sprintf("%v", uuid), line.Text)
if err != nil {
logger.Error("KafkaPublishMessageError", err.Error())
return err
}
return nil
}