forked from weaveworks/weave
-
Notifications
You must be signed in to change notification settings - Fork 0
/
connection.go
518 lines (466 loc) · 15.2 KB
/
connection.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
package router
import (
"encoding/binary"
"encoding/gob"
"fmt"
"log"
"net"
"sync"
"time"
)
type Connection interface {
Local() *Peer
Remote() *Peer
RemoteTCPAddr() string
Outbound() bool
Established() bool
BreakTie(Connection) ConnectionTieBreak
Shutdown(error)
Log(args ...interface{})
}
type ConnectionTieBreak int
const (
TieBreakWon ConnectionTieBreak = iota
TieBreakLost
TieBreakTied
)
type RemoteConnection struct {
local *Peer
remote *Peer
remoteTCPAddr string
outbound bool
established bool
}
type LocalConnection struct {
sync.RWMutex
RemoteConnection
TCPConn *net.TCPConn
tcpSender TCPSender
remoteUDPAddr *net.UDPAddr
receivedHeartbeat bool
stackFrag bool
effectivePMTU int
SessionKey *[32]byte
heartbeatTCP *time.Ticker
heartbeatTimeout *time.Timer
heartbeatFrame *ForwardedFrame
heartbeat *time.Ticker
fragTest *time.Ticker
forwarder *Forwarder
forwarderDF *ForwarderDF
Decryptor Decryptor
Router *Router
uid uint64
actionChan chan<- ConnectionAction
finished <-chan struct{} // closed to signal that actorLoop has finished
}
type ConnectionAction func() error
func NewRemoteConnection(from, to *Peer, tcpAddr string, outbound bool, established bool) *RemoteConnection {
return &RemoteConnection{
local: from,
remote: to,
remoteTCPAddr: tcpAddr,
outbound: outbound,
established: established,
}
}
func (conn *RemoteConnection) Local() *Peer { return conn.local }
func (conn *RemoteConnection) Remote() *Peer { return conn.remote }
func (conn *RemoteConnection) RemoteTCPAddr() string { return conn.remoteTCPAddr }
func (conn *RemoteConnection) Outbound() bool { return conn.outbound }
func (conn *RemoteConnection) Established() bool { return conn.established }
func (conn *RemoteConnection) BreakTie(Connection) ConnectionTieBreak { return TieBreakTied }
func (conn *RemoteConnection) Shutdown(error) {}
func (conn *RemoteConnection) Log(args ...interface{}) {
log.Println(append(append([]interface{}{}, fmt.Sprintf("->[%s|%s]:", conn.remoteTCPAddr, conn.remote)), args...)...)
}
func (conn *RemoteConnection) String() string {
from := "<nil>"
if conn.local != nil {
from = conn.local.String()
}
to := "<nil>"
if conn.remote != nil {
to = conn.remote.String()
}
return fmt.Sprint("Connection ", from, "->", to)
}
func NewLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router) *LocalConnection {
if connRemote.local != router.Ourself.Peer {
log.Fatal("Attempt to create local connection from a peer which is not ourself")
}
// NB, we're taking a copy of connRemote here.
return &LocalConnection{
RemoteConnection: *connRemote,
Router: router,
TCPConn: tcpConn,
remoteUDPAddr: udpAddr,
effectivePMTU: DefaultPMTU}
}
// Async. Does not return anything. If the connection is successful,
// it will end up in the local peer's connections map.
func (conn *LocalConnection) Start(acceptNewPeer bool) {
actionChan := make(chan ConnectionAction, ChannelSize)
conn.actionChan = actionChan
finished := make(chan struct{})
conn.finished = finished
go conn.run(actionChan, finished, acceptNewPeer)
}
func (conn *LocalConnection) BreakTie(dupConn Connection) ConnectionTieBreak {
dupConnLocal := dupConn.(*LocalConnection)
// conn.uid is used as the tie breaker here, in the knowledge that
// both sides will make the same decision.
if conn.uid < dupConnLocal.uid {
return TieBreakWon
} else if dupConnLocal.uid < conn.uid {
return TieBreakLost
} else {
return TieBreakTied
}
}
// Read by the forwarder processes when in the UDP senders
func (conn *LocalConnection) RemoteUDPAddr() *net.UDPAddr {
conn.RLock()
defer conn.RUnlock()
return conn.remoteUDPAddr
}
func (conn *LocalConnection) Established() bool {
conn.RLock()
defer conn.RUnlock()
return conn.established
}
// Called by forwarder processes, read in Forward (by sniffer and udp
// listener process in router).
func (conn *LocalConnection) setEffectivePMTU(pmtu int) {
conn.Lock()
defer conn.Unlock()
if conn.effectivePMTU != pmtu {
conn.effectivePMTU = pmtu
conn.Log("Effective PMTU set to", pmtu)
}
}
// Called by the connection's actor process, and by the connection's
// TCP received process. StackFrag is read in conn.Forward (called by
// router udp listener and sniffer processes)
func (conn *LocalConnection) setStackFrag(frag bool) {
conn.Lock()
defer conn.Unlock()
conn.stackFrag = frag
}
// Called by the connection's TCP receiver process.
func (conn *LocalConnection) pmtuVerified(pmtu int) {
conn.RLock()
fwd := conn.forwarderDF
conn.RUnlock()
if fwd != nil {
fwd.PMTUVerified(pmtu)
}
}
// Send directly, not via the Actor. If it goes via the Actor we can
// get a deadlock where LocalConnection is blocked talking to
// LocalPeer and LocalPeer is blocked trying send a ProtocolMsg via
// LocalConnection, and the channels are full in both directions so
// nothing can proceed.
func (conn *LocalConnection) SendProtocolMsg(m ProtocolMsg) {
if err := conn.sendProtocolMsg(m); err != nil {
conn.Shutdown(err)
}
}
// ACTOR methods
// NB: The conn.* fields are only written by the connection actor
// process, which is the caller of the ConnectionAction funs. Hence we
// do not need locks for reading, and only need write locks for fields
// read by other processes.
// Async
func (conn *LocalConnection) Shutdown(err error) {
// err should always be a real error, even if only io.EOF
if err == nil {
panic("nil error")
}
// Run on its own goroutine in case the channel is backed up
go func() { conn.sendAction(func() error { return err }) }()
}
// Async
//
// Heartbeating serves two purposes: a) keeping NAT paths alive, and
// b) updating a remote peer's knowledge of our address, in the event
// it changes (e.g. because NAT paths expired).
func (conn *LocalConnection) ReceivedHeartbeat(remoteUDPAddr *net.UDPAddr, connUID uint64) {
if remoteUDPAddr == nil || connUID != conn.uid {
return
}
conn.sendAction(func() error {
oldRemoteUDPAddr := conn.remoteUDPAddr
old := conn.receivedHeartbeat
conn.Lock()
conn.remoteUDPAddr = remoteUDPAddr
conn.receivedHeartbeat = true
conn.Unlock()
conn.heartbeatTimeout.Reset(HeartbeatTimeout)
if !old {
if err := conn.sendSimpleProtocolMsg(ProtocolConnectionEstablished); err != nil {
return err
}
}
if oldRemoteUDPAddr == nil {
return conn.sendFastHeartbeats()
} else if oldRemoteUDPAddr.String() != remoteUDPAddr.String() {
log.Println("Peer", conn.remote, "moved from", old, "to", remoteUDPAddr)
}
return nil
})
}
// Async
func (conn *LocalConnection) SetEstablished() {
conn.sendAction(func() error {
stopTicker(conn.heartbeat)
old := conn.established
conn.Lock()
conn.established = true
conn.Unlock()
if old {
return nil
}
conn.Router.Ourself.ConnectionEstablished(conn)
if err := conn.ensureForwarders(); err != nil {
return err
}
// Send a large frame down the DF channel in order to prompt
// PMTU discovery to start.
conn.Forward(true, &ForwardedFrame{
srcPeer: conn.local,
dstPeer: conn.remote,
frame: PMTUDiscovery},
nil)
conn.heartbeat = time.NewTicker(SlowHeartbeat)
conn.fragTest = time.NewTicker(FragTestInterval)
// avoid initial waits for timers to fire
conn.Forward(true, conn.heartbeatFrame, nil)
conn.setStackFrag(false)
if err := conn.sendSimpleProtocolMsg(ProtocolStartFragmentationTest); err != nil {
return err
}
return nil
})
}
// Send an actor request to the actorLoop, but don't block if
// actorLoop has exited - see http://blog.golang.org/pipelines for
// pattern
func (conn *LocalConnection) sendAction(action ConnectionAction) {
select {
case conn.actionChan <- action:
case <-conn.finished:
}
}
// ACTOR server
func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished chan<- struct{}, acceptNewPeer bool) {
var err error // important to use this var and not create another one with 'err :='
defer func() { conn.shutdown(err) }()
defer close(finished)
tcpConn := conn.TCPConn
tcpConn.SetLinger(0)
enc := gob.NewEncoder(tcpConn)
dec := gob.NewDecoder(tcpConn)
if err = conn.handshake(enc, dec, acceptNewPeer); err != nil {
return
}
conn.Log("completed handshake")
// The ordering of the following is very important. [1]
if conn.remoteUDPAddr != nil {
if err = conn.ensureForwarders(); err != nil {
return
}
}
if err = conn.Router.Ourself.AddConnection(conn); err != nil {
return
}
if err = conn.initHeartbeats(); err != nil {
return
}
go conn.receiveTCP(dec)
err = conn.actorLoop(actionChan)
}
// [1] Ordering constraints:
//
// (a) AddConnections must precede initHeartbeats. It is only after
// the former completes that we know the connection is valid, in
// particular is not a duplicate connection to the same peer. Sending
// heartbeats on a duplicate connection can trip up crypto at the
// other end, since the associated UDP packets may get decoded by the
// other connection. It is also generally wasteful to engage in any
// interaction with the remote on a connection that turns out to be
// invald.
//
// (b) AddConnection must precede receiveTCP. In the absence of any
// indirect connectivity to the remote peer, the first we hear about
// it (and any peers reachable from it) is through topology gossip it
// sends us on the connection. We must ensure that the connection has
// been added to Ourself prior to processing any such gossip,
// otherwise we risk immediately gc'ing part of that newly received
// portion of the topology (though not the remote peer itself, since
// that will have a positive ref count), leaving behind dangling
// references to peers. Hence we must invoke AddConnection, which is
// *synchronous*, first.
//
// (c) AddConnection must precede actorLoop. More precisely, it must
// precede shutdown, since that invokes DeleteConnection and is
// invoked on termination of this entire function. Essentially this
// boils down to a prohibition on running AddConnection in a separate
// goroutine, at least not without some synchronisation. Which in turn
// requires us the launching of the receiveTCP goroutine to precede
// actorLoop.
//
// (d) AddConnection should precede receiveTCP. There is no point
// starting the latter if the former fails.
//
// (e) initHeartbeats should precede actorLoop. The former is setting
// LocalConnection fields accessed by the latter. Since the latter
// runs in a separate goroutine, we'd have to add some synchronisation
// if initHeartbeats isn't run first.
//
// (f) ensureForwarders should precede AddConnection. As soon as a
// connection has been added to LocalPeer by the latter, it becomes
// visible to the packet routing logic, which will end up dropping
// packets if the forwarders haven't been created yet. We cannot
// prevent that completely, since, for example, forwarder can only be
// created when we know the remote UDP address, but it helps to try.
func (conn *LocalConnection) initHeartbeats() error {
conn.heartbeatTCP = time.NewTicker(TCPHeartbeat)
conn.heartbeatTimeout = time.NewTimer(HeartbeatTimeout)
heartbeatFrameBytes := make([]byte, EthernetOverhead+8)
binary.BigEndian.PutUint64(heartbeatFrameBytes[EthernetOverhead:], conn.uid)
conn.heartbeatFrame = &ForwardedFrame{
srcPeer: conn.local,
dstPeer: conn.remote,
frame: heartbeatFrameBytes}
if conn.remoteUDPAddr == nil {
return nil
}
return conn.sendFastHeartbeats()
}
func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err error) {
for err == nil {
select {
case action := <-actionChan:
err = action()
case <-conn.heartbeatTCP.C:
err = conn.sendSimpleProtocolMsg(ProtocolHeartbeat)
case <-conn.heartbeatTimeout.C:
err = fmt.Errorf("timed out waiting for UDP heartbeat")
case <-tickerChan(conn.heartbeat):
conn.Forward(true, conn.heartbeatFrame, nil)
case <-tickerChan(conn.fragTest):
conn.setStackFrag(false)
err = conn.sendSimpleProtocolMsg(ProtocolStartFragmentationTest)
}
}
return
}
func (conn *LocalConnection) shutdown(err error) {
if conn.remote == nil {
log.Printf("->[%s] connection shutting down due to error during handshake: %v\n", conn.remoteTCPAddr, err)
} else {
conn.Log("connection shutting down due to error:", err)
}
if conn.TCPConn != nil {
checkWarn(conn.TCPConn.Close())
}
if conn.remote != nil {
conn.Router.Peers.Dereference(conn.remote)
conn.Router.Ourself.DeleteConnection(conn)
}
if conn.heartbeatTimeout != nil {
conn.heartbeatTimeout.Stop()
}
stopTicker(conn.heartbeatTCP)
stopTicker(conn.heartbeat)
stopTicker(conn.fragTest)
// blank out the forwardChan so that the router processes don't
// try to send any more
conn.stopForwarders()
conn.Router.ConnectionMaker.ConnectionTerminated(conn.remoteTCPAddr, err)
}
// Helpers
func (conn *LocalConnection) sendSimpleProtocolMsg(tag ProtocolTag) error {
return conn.sendProtocolMsg(ProtocolMsg{tag: tag})
}
func (conn *LocalConnection) sendProtocolMsg(m ProtocolMsg) error {
return conn.tcpSender.Send(Concat([]byte{byte(m.tag)}, m.msg))
}
func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder) {
usingPassword := conn.SessionKey != nil
var receiver TCPReceiver
if usingPassword {
receiver = NewEncryptedTCPReceiver(conn.SessionKey, conn.outbound)
} else {
receiver = NewSimpleTCPReceiver()
}
var err error
for {
var msg []byte
conn.extendReadDeadline()
if err = decoder.Decode(&msg); err != nil {
break
}
msg, err = receiver.Decode(msg)
if err != nil {
break
}
if len(msg) < 1 {
conn.Log("ignoring blank msg")
continue
}
if err = conn.handleProtocolMsg(ProtocolTag(msg[0]), msg[1:]); err != nil {
break
}
}
conn.Shutdown(err)
}
func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte) error {
switch tag {
case ProtocolHeartbeat:
case ProtocolConnectionEstablished:
// We sent fast heartbeats to the remote peer, which has now
// received at least one of them and told us via this message.
// We can now consider the connection as established from our
// end.
conn.SetEstablished()
case ProtocolStartFragmentationTest:
conn.Forward(false, &ForwardedFrame{
srcPeer: conn.local,
dstPeer: conn.remote,
frame: FragTest},
nil)
case ProtocolFragmentationReceived:
conn.setStackFrag(true)
case ProtocolPMTUVerified:
conn.pmtuVerified(int(binary.BigEndian.Uint16(payload)))
case ProtocolGossipUnicast, ProtocolGossipBroadcast, ProtocolGossip:
return conn.Router.handleGossip(tag, payload)
default:
conn.Log("ignoring unknown protocol tag:", tag)
}
return nil
}
func (conn *LocalConnection) extendReadDeadline() {
conn.TCPConn.SetReadDeadline(time.Now().Add(TCPHeartbeat * 2))
}
func (conn *LocalConnection) sendFastHeartbeats() error {
err := conn.ensureForwarders()
if err == nil {
conn.heartbeat = time.NewTicker(FastHeartbeat)
conn.Forward(true, conn.heartbeatFrame, nil) // avoid initial wait
}
return err
}
func tickerChan(ticker *time.Ticker) <-chan time.Time {
if ticker != nil {
return ticker.C
}
return nil
}
func stopTicker(ticker *time.Ticker) {
if ticker != nil {
ticker.Stop()
}
}