forked from elastic/beats
-
Notifications
You must be signed in to change notification settings - Fork 7
/
sync.go
133 lines (109 loc) · 2.83 KB
/
sync.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
package logstash
import (
"time"
"github.com/elastic/go-lumber/client/v2"
"github.com/elastic/beats/libbeat/logp"
"github.com/elastic/beats/libbeat/outputs"
"github.com/elastic/beats/libbeat/outputs/transport"
)
const (
minWindowSize int = 1
defaultStartMaxWindowSize int = 10
)
type client struct {
*transport.Client
client *v2.SyncClient
win window
}
func newLumberjackClient(
conn *transport.Client,
compressLevel int,
maxWindowSize int,
timeout time.Duration,
beat string,
) (*client, error) {
c := &client{}
c.Client = conn
c.win.init(defaultStartMaxWindowSize, maxWindowSize)
enc, err := makeLogstashEventEncoder(beat)
if err != nil {
return nil, err
}
cl, err := v2.NewSyncClientWithConn(conn,
v2.JSONEncoder(enc),
v2.Timeout(timeout),
v2.CompressionLevel(compressLevel))
if err != nil {
return nil, err
}
c.client = cl
return c, nil
}
func (c *client) Connect(timeout time.Duration) error {
logp.Debug("logstash", "connect")
return c.Client.Connect()
}
func (c *client) Close() error {
logp.Debug("logstash", "close connection")
return c.Client.Close()
}
func (c *client) PublishEvent(data outputs.Data) error {
_, err := c.PublishEvents([]outputs.Data{data})
return err
}
// PublishEvents sends all events to logstash. On error a slice with all events
// not published or confirmed to be processed by logstash will be returned.
func (c *client) PublishEvents(
data []outputs.Data,
) ([]outputs.Data, error) {
publishEventsCallCount.Add(1)
totalNumberOfEvents := len(data)
for len(data) > 0 {
n, err := c.publishWindowed(data)
debug("%v events out of %v events sent to logstash. Continue sending",
n, len(data))
data = data[n:]
if err != nil {
c.win.shrinkWindow()
_ = c.Close()
logp.Err("Failed to publish events caused by: %v", err)
eventsNotAcked.Add(int64(len(data)))
ackedEvents.Add(int64(totalNumberOfEvents - len(data)))
return data, err
}
}
ackedEvents.Add(int64(totalNumberOfEvents))
return nil, nil
}
// publishWindowed published events with current maximum window size to logstash
// returning the total number of events sent (due to window size, or acks until
// failure).
func (c *client) publishWindowed(data []outputs.Data) (int, error) {
if len(data) == 0 {
return 0, nil
}
batchSize := len(data)
windowSize := c.win.get()
debug("Try to publish %v events to logstash with window size %v",
batchSize, windowSize)
// prepare message payload
if batchSize > windowSize {
data = data[:windowSize]
}
n, err := c.sendEvents(data)
if err != nil {
return n, err
}
c.win.tryGrowWindow(batchSize)
return len(data), nil
}
func (c *client) sendEvents(data []outputs.Data) (int, error) {
if len(data) == 0 {
return 0, nil
}
window := make([]interface{}, len(data))
for i, d := range data {
window[i] = d
}
return c.client.Send(window)
}