From fdc5c1091f4f7588369f525eb91f24a945d07d89 Mon Sep 17 00:00:00 2001 From: Wieger Steggerda Date: Tue, 2 Feb 2016 15:38:27 +0100 Subject: [PATCH] 1. make the update rules more transparent. 2. fixes bug: If a member is in the leave state, only updates where the member is made alive are apllied. This means that updates where the member changes state to the Suspect, Faulty or Leave state are ignored, even if the the updates should be aplied due to having a bigger incarnation number. --- events/test/mocks/event_listener.go | 1 + forward/mock_event_listener_test.go | 1 + forward/mock_logger_test.go | 29 +++++ forward/mock_sender_test.go | 3 + swim/member.go | 74 +++++------ swim/member_test.go | 121 ++++++++++++++++++ test/mocks/client_factory.go | 3 + test/mocks/logger.go | 29 +++++ test/mocks/ringpop.go | 25 ++++ test/mocks/stats_reporter.go | 5 + test/mocks/swim_node.go | 15 +++ test/mocks/t_chan_client.go | 1 + test/thrift/pingpong/mock_t_chan_ping_pong.go | 1 + 13 files changed, 271 insertions(+), 37 deletions(-) create mode 100644 swim/member_test.go diff --git a/events/test/mocks/event_listener.go b/events/test/mocks/event_listener.go index 4c206705..5533aaf4 100644 --- a/events/test/mocks/event_listener.go +++ b/events/test/mocks/event_listener.go @@ -7,6 +7,7 @@ type EventListener struct { mock.Mock } +// HandleEvent provides a mock function with given fields: event func (_m *EventListener) HandleEvent(event events.Event) { _m.Called(event) } diff --git a/forward/mock_event_listener_test.go b/forward/mock_event_listener_test.go index 4e3cb87e..5276a026 100644 --- a/forward/mock_event_listener_test.go +++ b/forward/mock_event_listener_test.go @@ -7,6 +7,7 @@ type EventListener struct { mock.Mock } +// HandleEvent provides a mock function with given fields: event func (_m *EventListener) HandleEvent(event events.Event) { _m.Called(event) } diff --git a/forward/mock_logger_test.go b/forward/mock_logger_test.go index add62c38..6e5907ef 100644 --- a/forward/mock_logger_test.go +++ b/forward/mock_logger_test.go @@ -7,42 +7,67 @@ type Logger struct { mock.Mock } +// Debug provides a mock function with given fields: args func (_m *Logger) Debug(args ...interface{}) { _m.Called(args) } + +// Debugf provides a mock function with given fields: format, args func (_m *Logger) Debugf(format string, args ...interface{}) { _m.Called(format, args) } + +// Info provides a mock function with given fields: args func (_m *Logger) Info(args ...interface{}) { _m.Called(args) } + +// Infof provides a mock function with given fields: format, args func (_m *Logger) Infof(format string, args ...interface{}) { _m.Called(format, args) } + +// Warn provides a mock function with given fields: args func (_m *Logger) Warn(args ...interface{}) { _m.Called(args) } + +// Warnf provides a mock function with given fields: format, args func (_m *Logger) Warnf(format string, args ...interface{}) { _m.Called(format, args) } + +// Error provides a mock function with given fields: args func (_m *Logger) Error(args ...interface{}) { _m.Called(args) } + +// Errorf provides a mock function with given fields: format, args func (_m *Logger) Errorf(format string, args ...interface{}) { _m.Called(format, args) } + +// Fatal provides a mock function with given fields: args func (_m *Logger) Fatal(args ...interface{}) { _m.Called(args) } + +// Fatalf provides a mock function with given fields: format, args func (_m *Logger) Fatalf(format string, args ...interface{}) { _m.Called(format, args) } + +// Panic provides a mock function with given fields: args func (_m *Logger) Panic(args ...interface{}) { _m.Called(args) } + +// Panicf provides a mock function with given fields: format, args func (_m *Logger) Panicf(format string, args ...interface{}) { _m.Called(format, args) } + +// WithField provides a mock function with given fields: key, value func (_m *Logger) WithField(key string, value interface{}) bark.Logger { ret := _m.Called(key, value) @@ -55,6 +80,8 @@ func (_m *Logger) WithField(key string, value interface{}) bark.Logger { return r0 } + +// WithFields provides a mock function with given fields: keyValues func (_m *Logger) WithFields(keyValues bark.LogFields) bark.Logger { ret := _m.Called(keyValues) @@ -67,6 +94,8 @@ func (_m *Logger) WithFields(keyValues bark.LogFields) bark.Logger { return r0 } + +// Fields provides a mock function with given fields: func (_m *Logger) Fields() bark.Fields { ret := _m.Called() diff --git a/forward/mock_sender_test.go b/forward/mock_sender_test.go index af823ffc..e2610f89 100644 --- a/forward/mock_sender_test.go +++ b/forward/mock_sender_test.go @@ -6,6 +6,7 @@ type MockSender struct { mock.Mock } +// WhoAmI provides a mock function with given fields: func (_m *MockSender) WhoAmI() (string, error) { ret := _m.Called() @@ -25,6 +26,8 @@ func (_m *MockSender) WhoAmI() (string, error) { return r0, r1 } + +// Lookup provides a mock function with given fields: _a0 func (_m *MockSender) Lookup(_a0 string) (string, error) { ret := _m.Called(_a0) diff --git a/swim/member.go b/swim/member.go index 116aac23..4bbe7eec 100644 --- a/swim/member.go +++ b/swim/member.go @@ -31,14 +31,14 @@ const ( // Alive is the member "alive" state Alive = "alive" + // Suspect is the member "suspect" state + Suspect = "suspect" + // Faulty is the member "faulty" state Faulty = "faulty" // Leave is the member "leave" state Leave = "leave" - - // Suspect is the member "suspect" state - Suspect = "suspect" ) // A Member is a member in the member list @@ -70,46 +70,46 @@ func shuffle(members []*Member) []*Member { return newMembers } +// nonLocalOverride returns wether a change should be applied to the member. +// This function assumes that the address of the member and the change are +// equal. func (m *Member) nonLocalOverride(change Change) bool { - return m.aliveOverride(change) || - m.suspectOverride(change) || - m.faultyOverride(change) || - m.leaveOverride(change) -} - -func (m *Member) aliveOverride(change Change) bool { - return change.Status == Alive && change.Incarnation > m.Incarnation -} - -func (m *Member) faultyOverride(change Change) bool { - return change.Status == Faulty && - ((m.Status == Suspect && change.Incarnation >= m.Incarnation) || - (m.Status == Faulty && change.Incarnation > m.Incarnation) || - (m.Status == Alive && change.Incarnation >= m.Incarnation)) -} + // change is younger than current member + if change.Incarnation > m.Incarnation { + return true + } -func (m *Member) leaveOverride(change Change) bool { - return change.Status == Leave && - m.Status != Leave && change.Incarnation >= m.Incarnation -} + // change is older than current member + if change.Incarnation < m.Incarnation { + return false + } -func (m *Member) suspectOverride(change Change) bool { - return change.Status == Suspect && - ((m.Status == Suspect && change.Incarnation > m.Incarnation) || - (m.Status == Faulty && change.Incarnation > m.Incarnation) || - (m.Status == Alive && change.Incarnation >= m.Incarnation)) + // If the incarnation numbers are equal, we look at the state to + // determine wether the change overrides this member. + return statePrecedence(change.Status) > statePrecedence(m.Status) } +// localOverride returns whether a change should be applied to to the member func (m *Member) localOverride(local string, change Change) bool { - return m.localSuspectOverride(local, change) || m.localFaultyOverride(local, change) -} - -func (m *Member) localFaultyOverride(local string, change Change) bool { - return m.Address == local && change.Status == Faulty -} - -func (m *Member) localSuspectOverride(local string, change Change) bool { - return m.Address == local && change.Status == Suspect + if m.Address != local { + return false + } + return change.Status == Faulty || change.Status == Suspect +} + +func statePrecedence(s string) int { + switch s { + case Alive: + return 0 + case Suspect: + return 1 + case Faulty: + return 2 + case Leave: + return 3 + default: + panic("invalid state") + } } func (m *Member) isReachable() bool { diff --git a/swim/member_test.go b/swim/member_test.go new file mode 100644 index 00000000..eaf7b2bf --- /dev/null +++ b/swim/member_test.go @@ -0,0 +1,121 @@ +// Copyright (c) 2016 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package swim + +import ( + "testing" + + "github.com/stretchr/testify/suite" + "github.com/uber/ringpop-go/util" +) + +type MemberTestSuite struct { + suite.Suite + states []state + localAddr string + nonLocalAddr string +} + +type state struct { + incNum int64 + status string +} + +func (s *MemberTestSuite) SetupTest() { + s.localAddr = "local address" + s.nonLocalAddr = "non-local address" + + incNumStart := util.TimeNowMS() + statuses := []string{Alive, Suspect, Faulty, Leave} + + // Add incNo, status combinations of ever increasing precedence. + s.states = nil + for i := int64(0); i < 4; i++ { + for _, status := range statuses { + s.states = append(s.states, state{incNumStart + i, status}) + } + } +} + +func newMember(addr string, s state) Member { + return Member{ + Address: addr, + Status: s.status, + Incarnation: s.incNum, + } +} + +func newChange(addr string, s state) Change { + return Change{ + Address: addr, + Status: s.status, + Incarnation: s.incNum, + } +} + +func (s *MemberTestSuite) TestNonLocalOverride() { + // NonLocalOverride ignores the locallity and only cares about the + // incarnation number and status of the members and changes. Since + // the state (incNum, status pairs) slice is generated with ever + // increasing precendence, changes with index j override members + // with index i if and only if j > i. + for i, s1 := range s.states { + for j, s2 := range s.states { + m := newMember(s.localAddr, s1) + c := newChange(s.localAddr, s2) + expected := j > i + got := m.nonLocalOverride(c) + s.Equal(expected, got, "expected override if and only if j > i") + + m = newMember(s.nonLocalAddr, s1) + c = newChange(s.nonLocalAddr, s2) + expected = j > i + got = m.nonLocalOverride(c) + s.Equal(expected, got, "expected override if and only if j > i") + } + } +} + +func (s *MemberTestSuite) TestLocalOverride() { + // LocalOverride aggressively marks updates as overrides, even if the + // incarnation number of the update is lower than the incarnation + // number of the local member. The Update function reincarnates the node + // when LocalOverride returns true. This very aggressive approach is likely + // to change in the near future. + for _, s1 := range s.states { + for _, s2 := range s.states { + m := newMember(s.localAddr, s1) + c := newChange(s.localAddr, s2) + expected := c.Status == Suspect || c.Status == Faulty + got := m.localOverride(s.localAddr, c) + s.Equal(expected, got, "expected override when change.Status is suspect or faulty") + + m = newMember(s.nonLocalAddr, s1) + c = newChange(s.nonLocalAddr, s2) + got = m.localOverride(s.localAddr, c) + s.False(got, "expected no override since member is not local") + } + } +} + +func TestMemberTestSuite(t *testing.T) { + suite.Run(t, new(MemberTestSuite)) +} diff --git a/test/mocks/client_factory.go b/test/mocks/client_factory.go index 232af849..36c3f2e1 100644 --- a/test/mocks/client_factory.go +++ b/test/mocks/client_factory.go @@ -8,6 +8,7 @@ type ClientFactory struct { mock.Mock } +// GetLocalClient provides a mock function with given fields: func (_m *ClientFactory) GetLocalClient() interface{} { ret := _m.Called() @@ -22,6 +23,8 @@ func (_m *ClientFactory) GetLocalClient() interface{} { return r0 } + +// MakeRemoteClient provides a mock function with given fields: client func (_m *ClientFactory) MakeRemoteClient(client thrift.TChanClient) interface{} { ret := _m.Called(client) diff --git a/test/mocks/logger.go b/test/mocks/logger.go index 3310ecb6..0580611c 100644 --- a/test/mocks/logger.go +++ b/test/mocks/logger.go @@ -7,42 +7,67 @@ type Logger struct { mock.Mock } +// Debug provides a mock function with given fields: args func (_m *Logger) Debug(args ...interface{}) { _m.Called(args) } + +// Debugf provides a mock function with given fields: format, args func (_m *Logger) Debugf(format string, args ...interface{}) { _m.Called(format, args) } + +// Info provides a mock function with given fields: args func (_m *Logger) Info(args ...interface{}) { _m.Called(args) } + +// Infof provides a mock function with given fields: format, args func (_m *Logger) Infof(format string, args ...interface{}) { _m.Called(format, args) } + +// Warn provides a mock function with given fields: args func (_m *Logger) Warn(args ...interface{}) { _m.Called(args) } + +// Warnf provides a mock function with given fields: format, args func (_m *Logger) Warnf(format string, args ...interface{}) { _m.Called(format, args) } + +// Error provides a mock function with given fields: args func (_m *Logger) Error(args ...interface{}) { _m.Called(args) } + +// Errorf provides a mock function with given fields: format, args func (_m *Logger) Errorf(format string, args ...interface{}) { _m.Called(format, args) } + +// Fatal provides a mock function with given fields: args func (_m *Logger) Fatal(args ...interface{}) { _m.Called(args) } + +// Fatalf provides a mock function with given fields: format, args func (_m *Logger) Fatalf(format string, args ...interface{}) { _m.Called(format, args) } + +// Panic provides a mock function with given fields: args func (_m *Logger) Panic(args ...interface{}) { _m.Called(args) } + +// Panicf provides a mock function with given fields: format, args func (_m *Logger) Panicf(format string, args ...interface{}) { _m.Called(format, args) } + +// WithField provides a mock function with given fields: key, value func (_m *Logger) WithField(key string, value interface{}) bark.Logger { ret := _m.Called(key, value) @@ -55,6 +80,8 @@ func (_m *Logger) WithField(key string, value interface{}) bark.Logger { return r0 } + +// WithFields provides a mock function with given fields: keyValues func (_m *Logger) WithFields(keyValues bark.LogFields) bark.Logger { ret := _m.Called(keyValues) @@ -67,6 +94,8 @@ func (_m *Logger) WithFields(keyValues bark.LogFields) bark.Logger { return r0 } + +// Fields provides a mock function with given fields: func (_m *Logger) Fields() bark.Fields { ret := _m.Called() diff --git a/test/mocks/ringpop.go b/test/mocks/ringpop.go index 6342a2cb..47b83c14 100644 --- a/test/mocks/ringpop.go +++ b/test/mocks/ringpop.go @@ -15,9 +15,12 @@ type Ringpop struct { mock.Mock } +// Destroy provides a mock function with given fields: func (_m *Ringpop) Destroy() { _m.Called() } + +// App provides a mock function with given fields: func (_m *Ringpop) App() string { ret := _m.Called() @@ -30,6 +33,8 @@ func (_m *Ringpop) App() string { return r0 } + +// WhoAmI provides a mock function with given fields: func (_m *Ringpop) WhoAmI() (string, error) { ret := _m.Called() @@ -49,6 +54,8 @@ func (_m *Ringpop) WhoAmI() (string, error) { return r0, r1 } + +// Uptime provides a mock function with given fields: func (_m *Ringpop) Uptime() (time.Duration, error) { ret := _m.Called() @@ -68,9 +75,13 @@ func (_m *Ringpop) Uptime() (time.Duration, error) { return r0, r1 } + +// RegisterListener provides a mock function with given fields: l func (_m *Ringpop) RegisterListener(l events.EventListener) { _m.Called(l) } + +// Bootstrap provides a mock function with given fields: opts func (_m *Ringpop) Bootstrap(opts *swim.BootstrapOptions) ([]string, error) { ret := _m.Called(opts) @@ -92,6 +103,8 @@ func (_m *Ringpop) Bootstrap(opts *swim.BootstrapOptions) ([]string, error) { return r0, r1 } + +// Checksum provides a mock function with given fields: func (_m *Ringpop) Checksum() (uint32, error) { ret := _m.Called() @@ -111,6 +124,8 @@ func (_m *Ringpop) Checksum() (uint32, error) { return r0, r1 } + +// Lookup provides a mock function with given fields: key func (_m *Ringpop) Lookup(key string) (string, error) { ret := _m.Called(key) @@ -130,6 +145,8 @@ func (_m *Ringpop) Lookup(key string) (string, error) { return r0, r1 } + +// LookupN provides a mock function with given fields: key, n func (_m *Ringpop) LookupN(key string, n int) ([]string, error) { ret := _m.Called(key, n) @@ -151,6 +168,8 @@ func (_m *Ringpop) LookupN(key string, n int) ([]string, error) { return r0, r1 } + +// GetReachableMembers provides a mock function with given fields: func (_m *Ringpop) GetReachableMembers() ([]string, error) { ret := _m.Called() @@ -172,6 +191,8 @@ func (_m *Ringpop) GetReachableMembers() ([]string, error) { return r0, r1 } + +// CountReachableMembers provides a mock function with given fields: func (_m *Ringpop) CountReachableMembers() (int, error) { ret := _m.Called() @@ -191,6 +212,8 @@ func (_m *Ringpop) CountReachableMembers() (int, error) { return r0, r1 } + +// HandleOrForward provides a mock function with given fields: key, request, response, service, endpoint, format, opts func (_m *Ringpop) HandleOrForward(key string, request []byte, response *[]byte, service string, endpoint string, format tchannel.Format, opts *forward.Options) (bool, error) { ret := _m.Called(key, request, response, service, endpoint, format, opts) @@ -210,6 +233,8 @@ func (_m *Ringpop) HandleOrForward(key string, request []byte, response *[]byte, return r0, r1 } + +// Forward provides a mock function with given fields: dest, keys, request, service, endpoint, format, opts func (_m *Ringpop) Forward(dest string, keys []string, request []byte, service string, endpoint string, format tchannel.Format, opts *forward.Options) ([]byte, error) { ret := _m.Called(dest, keys, request, service, endpoint, format, opts) diff --git a/test/mocks/stats_reporter.go b/test/mocks/stats_reporter.go index 3a2bdb7e..cfbc7d8a 100644 --- a/test/mocks/stats_reporter.go +++ b/test/mocks/stats_reporter.go @@ -9,12 +9,17 @@ type StatsReporter struct { mock.Mock } +// IncCounter provides a mock function with given fields: name, tags, value func (_m *StatsReporter) IncCounter(name string, tags bark.Tags, value int64) { _m.Called(name, tags, value) } + +// UpdateGauge provides a mock function with given fields: name, tags, value func (_m *StatsReporter) UpdateGauge(name string, tags bark.Tags, value int64) { _m.Called(name, tags, value) } + +// RecordTimer provides a mock function with given fields: name, tags, d func (_m *StatsReporter) RecordTimer(name string, tags bark.Tags, d time.Duration) { _m.Called(name, tags, d) } diff --git a/test/mocks/swim_node.go b/test/mocks/swim_node.go index caf131be..f07017fd 100644 --- a/test/mocks/swim_node.go +++ b/test/mocks/swim_node.go @@ -7,6 +7,7 @@ type SwimNode struct { mock.Mock } +// Bootstrap provides a mock function with given fields: opts func (_m *SwimNode) Bootstrap(opts *swim.BootstrapOptions) ([]string, error) { ret := _m.Called(opts) @@ -28,6 +29,8 @@ func (_m *SwimNode) Bootstrap(opts *swim.BootstrapOptions) ([]string, error) { return r0, r1 } + +// CountReachableMembers provides a mock function with given fields: func (_m *SwimNode) CountReachableMembers() int { ret := _m.Called() @@ -40,9 +43,13 @@ func (_m *SwimNode) CountReachableMembers() int { return r0 } + +// Destroy provides a mock function with given fields: func (_m *SwimNode) Destroy() { _m.Called() } + +// GetReachableMembers provides a mock function with given fields: func (_m *SwimNode) GetReachableMembers() []string { ret := _m.Called() @@ -57,6 +64,8 @@ func (_m *SwimNode) GetReachableMembers() []string { return r0 } + +// MemberStats provides a mock function with given fields: func (_m *SwimNode) MemberStats() swim.MemberStats { ret := _m.Called() @@ -69,6 +78,8 @@ func (_m *SwimNode) MemberStats() swim.MemberStats { return r0 } + +// ProtocolStats provides a mock function with given fields: func (_m *SwimNode) ProtocolStats() swim.ProtocolStats { ret := _m.Called() @@ -81,6 +92,8 @@ func (_m *SwimNode) ProtocolStats() swim.ProtocolStats { return r0 } + +// Ready provides a mock function with given fields: func (_m *SwimNode) Ready() bool { ret := _m.Called() @@ -93,6 +106,8 @@ func (_m *SwimNode) Ready() bool { return r0 } + +// RegisterListener provides a mock function with given fields: l func (_m *SwimNode) RegisterListener(l swim.EventListener) { _m.Called(l) } diff --git a/test/mocks/t_chan_client.go b/test/mocks/t_chan_client.go index 8c6fec86..75aebd3b 100644 --- a/test/mocks/t_chan_client.go +++ b/test/mocks/t_chan_client.go @@ -9,6 +9,7 @@ type TChanClient struct { mock.Mock } +// Call provides a mock function with given fields: ctx, serviceName, methodName, req, resp func (_m *TChanClient) Call(ctx thrift.Context, serviceName string, methodName string, req athrift.TStruct, resp athrift.TStruct) (bool, error) { ret := _m.Called(ctx, serviceName, methodName, req, resp) diff --git a/test/thrift/pingpong/mock_t_chan_ping_pong.go b/test/thrift/pingpong/mock_t_chan_ping_pong.go index 05a0c64c..423742b1 100644 --- a/test/thrift/pingpong/mock_t_chan_ping_pong.go +++ b/test/thrift/pingpong/mock_t_chan_ping_pong.go @@ -8,6 +8,7 @@ type MockTChanPingPong struct { mock.Mock } +// Ping provides a mock function with given fields: ctx, request func (_m *MockTChanPingPong) Ping(ctx thrift.Context, request *Ping) (*Pong, error) { ret := _m.Called(ctx, request)