-
Notifications
You must be signed in to change notification settings - Fork 18
/
socket_server.go
192 lines (162 loc) · 4.96 KB
/
socket_server.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
// Copyright (c) 2022 Gobalsky Labs Limited
//
// Use of this software is governed by the Business Source License included
// in the LICENSE.DATANODE file and at https://www.mariadb.com/bsl11.
//
// Change Date: 18 months from the later of the date of the first publicly
// available Distribution of this version of the repository, and 25 June 2022.
//
// On the date above, in accordance with the Business Source License, use
// of this software will be governed by version 3 or later of the GNU General
// Public License.
package broker
import (
"context"
"fmt"
"net"
"strings"
"code.vegaprotocol.io/vega/core/events"
"code.vegaprotocol.io/vega/logging"
eventspb "code.vegaprotocol.io/vega/protos/vega/events/v1"
"golang.org/x/sync/errgroup"
"github.com/golang/protobuf/proto"
mangos "go.nanomsg.org/mangos/v3"
mangosErr "go.nanomsg.org/mangos/v3/errors"
"go.nanomsg.org/mangos/v3/protocol"
"go.nanomsg.org/mangos/v3/protocol/pull"
_ "go.nanomsg.org/mangos/v3/transport/inproc" // changes behavior of nanomsg
_ "go.nanomsg.org/mangos/v3/transport/tcp" // changes behavior of nanomsg
)
// socketServer receives events from a remote broker.
// This is used by the data node to receive events from a non-validating core node.
type socketServer struct {
log *logging.Logger
config *Config
sock protocol.Socket
}
func pipeEventToString(pe mangos.PipeEvent) string {
switch pe {
case mangos.PipeEventAttached:
return "Attached"
case mangos.PipeEventDetached:
return "Detached"
default:
return "Attaching"
}
}
func newSocketServer(log *logging.Logger, config *Config) (*socketServer, error) {
sock, err := pull.NewSocket()
if err != nil {
return nil, fmt.Errorf("failed to create new socket: %w", err)
}
return &socketServer{
log: log,
config: config,
sock: sock,
}, nil
}
func (s socketServer) Listen() error {
addr := fmt.Sprintf(
"%s://%s",
strings.ToLower(s.config.SocketConfig.TransportType),
net.JoinHostPort(s.config.SocketConfig.IP, fmt.Sprintf("%d", s.config.SocketConfig.Port)),
)
listenOptions := map[string]interface{}{mangos.OptionMaxRecvSize: 0}
listener, err := s.sock.NewListener(addr, listenOptions)
if err != nil {
return fmt.Errorf("failed to make listener %w", err)
}
if err := listener.Listen(); err != nil {
return fmt.Errorf("failed to listen on %v: %w", addr, err)
}
s.log.Info("Starting broker socket server", logging.String("addr", s.config.SocketConfig.IP),
logging.Int("port", s.config.SocketConfig.Port))
s.sock.SetPipeEventHook(func(pe mangos.PipeEvent, p mangos.Pipe) {
s.log.Info(
"New broker connection event",
logging.String("eventType", pipeEventToString(pe)),
logging.Uint32("id", p.ID()),
logging.String("address", p.Address()),
)
})
return nil
}
func (s socketServer) Receive(ctx context.Context) (<-chan events.Event, <-chan error) {
outboundCh := make(chan events.Event, s.config.SocketServerOutboundBufferSize)
// channel onto which we push the raw messages from the queue
inboundCh := make(chan []byte, s.config.SocketServerInboundBufferSize)
errCh := make(chan error, 1)
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
<-ctx.Done()
if err := s.close(); err != nil {
return fmt.Errorf("failed to close socket: %w", err)
}
return nil
})
eg.Go(func() error {
defer close(outboundCh)
for msg := range inboundCh {
var be eventspb.BusEvent
if err := proto.Unmarshal(msg, &be); err != nil {
// surely we should stop if this happens?
s.log.Error("Failed to unmarshal received event", logging.Error(err))
continue
}
if be.Version != eventspb.Version {
return fmt.Errorf("mismatched BusEvent version received: %d, want %d", be.Version, eventspb.Version)
}
evt := toEvent(ctx, &be)
if evt == nil {
s.log.Error("Can not convert proto event to internal event", logging.String("event_type", be.GetType().String()))
continue
}
// Listen for context cancels, even if we're blocked sending events
select {
case outboundCh <- evt:
case <-ctx.Done():
return ctx.Err()
}
}
return nil
})
eg.Go(func() error {
var recvTimeouts int
defer close(inboundCh)
for {
msg, err := s.sock.Recv()
if err != nil {
switch err {
case mangosErr.ErrRecvTimeout:
s.log.Warn("Receive socket timeout", logging.Error(err))
recvTimeouts++
if recvTimeouts > s.config.SocketConfig.MaxReceiveTimeouts {
return fmt.Errorf("more then a 3 socket timeouts occurred: %w", err)
}
case mangosErr.ErrBadVersion:
return fmt.Errorf("failed with bad protocol version: %w", err)
case mangosErr.ErrClosed:
return nil
default:
s.log.Error("Failed to Receive message", logging.Error(err))
continue
}
}
inboundCh <- msg
recvTimeouts = 0
}
})
go func() {
defer func() {
close(errCh)
}()
if err := eg.Wait(); err != nil {
errCh <- err
}
}()
return outboundCh, errCh
}
func (s socketServer) close() error {
s.log.Info("Closing socket server")
return s.sock.Close()
}