-
Notifications
You must be signed in to change notification settings - Fork 10
/
buffer.go
310 lines (284 loc) · 8.43 KB
/
buffer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
package buffer
import (
"errors"
"fmt"
"sort"
"time"
"github.com/openware/pkg/asset"
"github.com/openware/pkg/currency"
"github.com/openware/pkg/log"
"github.com/openware/pkg/orderbook"
)
const packageError = "websocket orderbook buffer error: %w"
var (
errUnsetExchangeName = errors.New("exchange name unset")
errUnsetDataHandler = errors.New("datahandler unset")
errIssueBufferEnabledButNoLimit = errors.New("buffer enabled but no limit set")
errUpdateIsNil = errors.New("update is nil")
errUpdateNoTargets = errors.New("update bid/ask targets cannot be nil")
errDepthNotFound = errors.New("orderbook depth not found")
errRESTOverwrite = errors.New("orderbook has been overwritten by REST protocol")
)
// Setup sets private variables
func (w *Orderbook) Setup(obBufferLimit int,
bufferEnabled,
sortBuffer,
sortBufferByUpdateIDs,
updateEntriesByID,
verbose bool,
exchangeName string,
dataHandler chan interface{}) error {
if exchangeName == "" {
return fmt.Errorf(packageError, errUnsetExchangeName)
}
if dataHandler == nil {
return fmt.Errorf(packageError, errUnsetDataHandler)
}
if bufferEnabled && obBufferLimit < 1 {
return fmt.Errorf(packageError, errIssueBufferEnabledButNoLimit)
}
w.obBufferLimit = obBufferLimit
w.bufferEnabled = bufferEnabled
w.sortBuffer = sortBuffer
w.sortBufferByUpdateIDs = sortBufferByUpdateIDs
w.updateEntriesByID = updateEntriesByID
w.exchangeName = exchangeName
w.dataHandler = dataHandler
w.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
w.verbose = verbose
return nil
}
// validate validates update against setup values
func (w *Orderbook) validate(u *Update) error {
if u == nil {
return fmt.Errorf(packageError, errUpdateIsNil)
}
if len(u.Bids) == 0 && len(u.Asks) == 0 {
return fmt.Errorf(packageError, errUpdateNoTargets)
}
return nil
}
// Update updates a stored pointer to an orderbook.Depth struct containing a
// linked list, this switches between the usage of a buffered update
func (w *Orderbook) Update(u *Update) error {
if err := w.validate(u); err != nil {
return err
}
w.m.Lock()
defer w.m.Unlock()
book, ok := w.ob[u.Pair.Base][u.Pair.Quote][u.Asset]
if !ok {
return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s",
errDepthNotFound,
w.exchangeName,
u.Pair,
u.Asset)
}
// Checks for when the rest protocol overwrites a streaming dominated book
// will stop updating book via incremental updates. This occurs because our
// sync manager (engine/sync.go) timer has elapsed for streaming. Usually
// because the book is highly illiquid. TODO: Book resubscribe on websocket.
if book.ob.IsRestSnapshot() {
if w.verbose {
log.Warnf(log.WebsocketMgr,
"%s for Exchange %s CurrencyPair: %s AssetType: %s consider extending synctimeoutwebsocket",
errRESTOverwrite,
w.exchangeName,
u.Pair,
u.Asset)
}
return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s",
errRESTOverwrite,
w.exchangeName,
u.Pair,
u.Asset)
}
// Apply new update information
book.ob.SetLastUpdate(u.UpdateTime, u.UpdateID, false)
if w.bufferEnabled {
processed, err := w.processBufferUpdate(book, u)
if err != nil {
return err
}
if !processed {
return nil
}
} else {
err := w.processObUpdate(book, u)
if err != nil {
return err
}
}
if book.ob.VerifyOrderbook { // This is used here so as to not retrieve
// book if verification is off.
// On every update, this will retrieve and verify orderbook depths
err := book.ob.Retrieve().Verify()
if err != nil {
return err
}
}
select {
case <-book.ticker.C:
// Opted to wait for receiver because we are limiting here and the sync
// manager requires update
go func() {
w.dataHandler <- book.ob.Retrieve()
book.ob.Publish()
}()
default:
// We do not need to send an update to the sync manager within this time
// window unless verbose is turned on
if w.verbose {
w.dataHandler <- book.ob.Retrieve()
book.ob.Publish()
}
}
return nil
}
// processBufferUpdate stores update into buffer, when buffer at capacity as
// defined by w.obBufferLimit it well then sort and apply updates.
func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *Update) (bool, error) {
*o.buffer = append(*o.buffer, *u)
if len(*o.buffer) < w.obBufferLimit {
return false, nil
}
if w.sortBuffer {
// sort by last updated to ensure each update is in order
if w.sortBufferByUpdateIDs {
sort.Slice(*o.buffer, func(i, j int) bool {
return (*o.buffer)[i].UpdateID < (*o.buffer)[j].UpdateID
})
} else {
sort.Slice(*o.buffer, func(i, j int) bool {
return (*o.buffer)[i].UpdateTime.Before((*o.buffer)[j].UpdateTime)
})
}
}
for i := range *o.buffer {
err := w.processObUpdate(o, &(*o.buffer)[i])
if err != nil {
return false, err
}
}
// clear buffer of old updates
*o.buffer = nil
return true, nil
}
// processObUpdate processes updates either by its corresponding id or by
// price level
func (w *Orderbook) processObUpdate(o *orderbookHolder, u *Update) error {
if w.updateEntriesByID {
return o.updateByIDAndAction(u)
}
o.updateByPrice(u)
return nil
}
// updateByPrice ammends amount if match occurs by price, deletes if amount is
// zero or less and inserts if not found.
func (o *orderbookHolder) updateByPrice(updts *Update) {
o.ob.UpdateBidAskByPrice(updts.Bids, updts.Asks, updts.MaxDepth)
}
// updateByIDAndAction will receive an action to execute against the orderbook
// it will then match by IDs instead of price to perform the action
func (o *orderbookHolder) updateByIDAndAction(updts *Update) error {
switch updts.Action {
case Amend:
return o.ob.UpdateBidAskByID(updts.Bids, updts.Asks)
case Delete:
// edge case for Bitfinex as their streaming endpoint duplicates deletes
bypassErr := o.ob.GetName() == "Bitfinex" && o.ob.IsFundingRate()
return o.ob.DeleteBidAskByID(updts.Bids, updts.Asks, bypassErr)
case Insert:
return o.ob.InsertBidAskByID(updts.Bids, updts.Asks)
case UpdateInsert:
return o.ob.UpdateInsertByID(updts.Bids, updts.Asks)
default:
return fmt.Errorf("invalid action [%s]", updts.Action)
}
}
// LoadSnapshot loads initial snapshot of orderbook data from websocket
func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error {
w.m.Lock()
defer w.m.Unlock()
m1, ok := w.ob[book.Pair.Base]
if !ok {
m1 = make(map[currency.Code]map[asset.Item]*orderbookHolder)
w.ob[book.Pair.Base] = m1
}
m2, ok := m1[book.Pair.Quote]
if !ok {
m2 = make(map[asset.Item]*orderbookHolder)
m1[book.Pair.Quote] = m2
}
holder, ok := m2[book.Asset]
if !ok {
// Associate orderbook pointer with local exchange depth map
depth, err := orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset)
if err != nil {
return err
}
depth.AssignOptions(book)
buffer := make([]Update, w.obBufferLimit)
ticker := time.NewTicker(timerDefault)
holder = &orderbookHolder{
ob: depth,
buffer: &buffer,
ticker: ticker,
}
m2[book.Asset] = holder
}
// Checks if book can deploy to linked list
err := book.Verify()
if err != nil {
return err
}
holder.ob.LoadSnapshot(book.Bids, book.Asks)
if holder.ob.VerifyOrderbook { // This is used here so as to not retrieve
// book if verification is off.
// Checks to see if orderbook snapshot that was deployed has not been
// altered in any way
err = holder.ob.Retrieve().Verify()
if err != nil {
return err
}
}
w.dataHandler <- holder.ob.Retrieve()
holder.ob.Publish()
return nil
}
// GetOrderbook returns an orderbook copy as orderbook.Base
func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) (*orderbook.Base, error) {
w.m.Lock()
defer w.m.Unlock()
book, ok := w.ob[p.Base][p.Quote][a]
if !ok {
return nil, fmt.Errorf("%s %s %s %w",
w.exchangeName,
p,
a,
errDepthNotFound)
}
return book.ob.Retrieve(), nil
}
// FlushBuffer flushes w.ob data to be garbage collected and refreshed when a
// connection is lost and reconnected
func (w *Orderbook) FlushBuffer() {
w.m.Lock()
w.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
w.m.Unlock()
}
// FlushOrderbook flushes independent orderbook
func (w *Orderbook) FlushOrderbook(p currency.Pair, a asset.Item) error {
w.m.Lock()
defer w.m.Unlock()
book, ok := w.ob[p.Base][p.Quote][a]
if !ok {
return fmt.Errorf("cannot flush orderbook %s %s %s %w",
w.exchangeName,
p,
a,
errDepthNotFound)
}
book.ob.Flush()
return nil
}