From 2a21f1ec93ed203537b039c6f3d0a17e39553d79 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Thu, 18 Jun 2020 16:51:54 +0900 Subject: [PATCH 01/15] =cluster,membership prepared synchronous gossip simulation spec --- .../Cluster+MembershipGossipLogic.swift | 10 +- .../Gossip/PeerSelection.swift | 36 ++- .../ActorTestKit.swift | 4 +- .../ClusteredActorSystemsXCTestCase.swift | 13 +- .../DistributedActorsTestKit/LogCapture.swift | 11 +- .../ActorSystem+Testing.swift | 11 +- .../Cluster/MembershipGossipLogicTests.swift | 271 ++++++++++++++++++ 7 files changed, 335 insertions(+), 21 deletions(-) create mode 100644 Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift index 6af79fdc6..7bfe5c040 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift @@ -17,13 +17,13 @@ import NIO // ==== ---------------------------------------------------------------------------------------------------------------- // MARK: Membership Gossip Logic -final class MembershipGossipLogic: GossipLogic { +final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { typealias Envelope = Cluster.Gossip private let context: Context - private lazy var localNode: UniqueNode = self.context.system.cluster.node + internal lazy var localNode: UniqueNode = self.context.system.cluster.node - private var latestGossip: Cluster.Gossip + internal var latestGossip: Cluster.Gossip private let notifyOnGossipRef: ActorRef init(_ context: Context, notifyOnGossipRef: ActorRef) { @@ -126,6 +126,10 @@ final class MembershipGossipLogic: GossipLogic { _ = self.latestGossip.mergeForward(incoming: incoming) // effects are signalled via the ClusterShell, not here (it will also perform a merge) // TODO: a bit duplicated, could we maintain it here? } + + var description: String { + "MembershipGossipLogic(\(localNode))" + } } // ==== ---------------------------------------------------------------------------------------------------------------- diff --git a/Sources/DistributedActors/Gossip/PeerSelection.swift b/Sources/DistributedActors/Gossip/PeerSelection.swift index f82621929..e8acf1ac2 100644 --- a/Sources/DistributedActors/Gossip/PeerSelection.swift +++ b/Sources/DistributedActors/Gossip/PeerSelection.swift @@ -21,9 +21,43 @@ /// // TODO: implement SWIMs selection in terms of this public protocol PeerSelection { associatedtype Peer: Hashable - typealias Peers = Array.SubSequence + typealias Peers = [Peer] func onMembershipEvent(event: Cluster.Event) func select() -> Peers } + +//public struct StableRandomRoundRobin { +// +// var peerSet: Set +// var peers: [Peer] +// +// // how many peers we select in each gossip round, +// // we could for example be dynamic and notice if we have 10+ nodes, we pick 2 members to speed up the dissemination etc. +// let n = 1 +// +// public init() { +// } +// +// func onMembershipEvent(event: Cluster.Event) { +// +// } +// +// func update(peers newPeers: [Peer]) { +// let newPeerSet = Set(peers) +// } +// +// func select() -> [Peer] { +// var selectedPeers: [AddressableActorRef] = [] +// selectedPeers.reserveCapacity(n) +// +// for peer in peers.shuffled() +// where selectedPeers.count < n && self.shouldGossipWith(peer) { +// selectedPeers.append(peer) +// } +// +// return selectedPeers +// } +// +//} diff --git a/Sources/DistributedActorsTestKit/ActorTestKit.swift b/Sources/DistributedActorsTestKit/ActorTestKit.swift index 5d88e729f..84ef86a9c 100644 --- a/Sources/DistributedActorsTestKit/ActorTestKit.swift +++ b/Sources/DistributedActorsTestKit/ActorTestKit.swift @@ -328,14 +328,14 @@ extension ActorTestKit { public extension ActorTestKit { /// Creates a _fake_ `ActorContext` which can be used to pass around to fulfil type argument requirements, /// however it DOES NOT have the ability to perform any of the typical actor context actions (such as spawning etc). - func makeFakeContext(forType: M.Type = M.self) -> ActorContext { + func makeFakeContext(of: M.Type = M.self) -> ActorContext { MockActorContext(self.system) } /// Creates a _fake_ `ActorContext` which can be used to pass around to fulfil type argument requirements, /// however it DOES NOT have the ability to perform any of the typical actor context actions (such as spawning etc). func makeFakeContext(for: Behavior) -> ActorContext { - self.makeFakeContext(forType: M.self) + self.makeFakeContext(of: M.self) } } diff --git a/Sources/DistributedActorsTestKit/Cluster/ClusteredActorSystemsXCTestCase.swift b/Sources/DistributedActorsTestKit/Cluster/ClusteredActorSystemsXCTestCase.swift index 14813bb5e..7505a5ac0 100644 --- a/Sources/DistributedActorsTestKit/Cluster/ClusteredActorSystemsXCTestCase.swift +++ b/Sources/DistributedActorsTestKit/Cluster/ClusteredActorSystemsXCTestCase.swift @@ -60,16 +60,18 @@ open class ClusteredActorSystemsXCTestCase: XCTestCase { /// Set up a new node intended to be clustered. open func setUpNode(_ name: String, _ modifySettings: ((inout ActorSystemSettings) -> Void)? = nil) -> ActorSystem { - var captureSettings = LogCapture.Settings() - self.configureLogCapture(settings: &captureSettings) - let capture = LogCapture(settings: captureSettings) - let node = ActorSystem(name) { settings in settings.cluster.enabled = true settings.cluster.node.port = self.nextPort() if self.captureLogs { + var captureSettings = LogCapture.Settings() + self.configureLogCapture(settings: &captureSettings) + let capture = LogCapture(settings: captureSettings) + settings.logging.logger = capture.logger(label: name) + + self._logCaptures.append(capture) } settings.cluster.autoLeaderElection = .lowestReachable(minNumberOfMembers: 2) @@ -85,9 +87,6 @@ open class ClusteredActorSystemsXCTestCase: XCTestCase { self._nodes.append(node) self._testKits.append(.init(node)) - if self.captureLogs { - self._logCaptures.append(capture) - } return node } diff --git a/Sources/DistributedActorsTestKit/LogCapture.swift b/Sources/DistributedActorsTestKit/LogCapture.swift index 638b31fff..9282d00ea 100644 --- a/Sources/DistributedActorsTestKit/LogCapture.swift +++ b/Sources/DistributedActorsTestKit/LogCapture.swift @@ -22,7 +22,6 @@ import XCTest /// /// ### Warning /// This handler uses locks for each and every operation. -// TODO: the implementation is quite incomplete and does not allow inspecting metadata setting etc. public final class LogCapture { private var _logs: [CapturedLogMessage] = [] private let lock = DistributedActorsConcurrencyHelpers.Lock() @@ -84,6 +83,11 @@ extension LogCapture { public var excludeGrep: Set = [] public var grep: Set = [] + public var ignoredMetadata: Set = [ + "actor/node", + "actor/nodeName", + ] + public init() {} } } @@ -110,8 +114,9 @@ extension LogCapture { } metadata.removeValue(forKey: "label") - metadata.removeValue(forKey: "actor/node") - metadata.removeValue(forKey: "actor/nodeName") + self.settings.ignoredMetadata.forEach { ignoreKey in + metadata.removeValue(forKey: ignoreKey) + } if !metadata.isEmpty { metadataString = "\n// metadata:\n" for key in metadata.keys.sorted() { diff --git a/Tests/DistributedActorsTests/ActorSystem+Testing.swift b/Tests/DistributedActorsTests/ActorSystem+Testing.swift index 3f0857642..01499a5fe 100644 --- a/Tests/DistributedActorsTests/ActorSystem+Testing.swift +++ b/Tests/DistributedActorsTests/ActorSystem+Testing.swift @@ -34,13 +34,14 @@ extension ActorSystem { /// Internal utility to create "known remote ref" on known target system. /// Real applications should never do this, and instead rely on the `Receptionist` to discover references. func _resolveKnownRemote(_ ref: ActorRef, onRemoteSystem remote: ActorSystem) -> ActorRef { - guard self._cluster != nil else { - fatalError("system must be clustered to allow resolving a remote ref.") - } + self._resolveKnownRemote(ref, onRemoteNode: remote.cluster.node) + } + + func _resolveKnownRemote(_ ref: ActorRef, onRemoteNode remoteNode: UniqueNode) -> ActorRef { guard let shell = self._cluster else { - fatalError("system._cluster shell must be available, was the resolve invoked too early (before system startup completed)?") + fatalError("Actor System must have clustering enabled to allow resolving remote actors") } - let remoteAddress = ActorAddress(node: remote.settings.cluster.uniqueBindNode, path: ref.path, incarnation: ref.address.incarnation) + let remoteAddress = ActorAddress(node: remoteNode, path: ref.path, incarnation: ref.address.incarnation) return ActorRef(.remote(RemoteClusterActorPersonality(shell: shell, address: remoteAddress, system: self))) } } diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift new file mode 100644 index 000000000..e66dd126b --- /dev/null +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift @@ -0,0 +1,271 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Distributed Actors open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the Swift Distributed Actors project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.md for the list of Swift Distributed Actors project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@testable import DistributedActors +import DistributedActorsTestKit +import NIO +import XCTest + +final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { + override func configureActorSystem(settings: inout ActorSystemSettings) { + settings.cluster.enabled = false // not actually clustering, just need a few nodes + } + + override func configureLogCapture(settings: inout LogCapture.Settings) { + settings.excludeActorPaths = [ + "/system/replicator/gossip", + "/system/replicator", + "/system/swim", + "/system/clusterEvents", + "/system/cluster", + "/system/leadership", + ] + } + + lazy var systemA: ActorSystem! = nil + lazy var systemB: ActorSystem! = nil + lazy var systemC: ActorSystem! = nil + var systems: [ActorSystem] = [] + var nodes: [UniqueNode] = [] + var allPeers: [AddressableActorRef] = [] + + lazy var a: AddressableActorRef = self.allPeers.first { $0.address.node?.node.systemName == "A" }! + lazy var b: AddressableActorRef = self.allPeers.first { $0.address.node?.node.systemName == "B" }! + lazy var c: AddressableActorRef = self.allPeers.first { $0.address.node?.node.systemName == "C" }! + + lazy var testKit: ActorTestKit! = nil + + lazy var pA: ActorTestProbe! = nil + lazy var logicA: MembershipGossipLogic! = nil + + lazy var pB: ActorTestProbe! = nil + lazy var logicB: MembershipGossipLogic! = nil + + lazy var pC: ActorTestProbe! = nil + lazy var logicC: MembershipGossipLogic! = nil + + var logics: [MembershipGossipLogic] { + [self.logicA, self.logicB, self.logicC] + } + + override func setUp() { + super.setUp() + self.systemA = setUpNode("A") { settings in + settings.cluster.enabled = true // to allow remote resolves, though we never send messages there + } + self.systemB = setUpNode("B") + self.systemC = setUpNode("C") + + self.systems = [systemA, systemB, systemC] + self.nodes = systems.map { $0.cluster.node } + self.allPeers = try! systems.map { system -> ActorRef.Message> in + let ref: ActorRef.Message> = try system.spawn("peer", .receiveMessage { _ in .same }) + return self.systemA._resolveKnownRemote(ref, onRemoteSystem: system) + }.map { $0.asAddressable() } + + self.testKit = self.testKit(self.systemA) + + self.pA = testKit.spawnTestProbe(expecting: Cluster.Gossip.self) + self.pB = testKit.spawnTestProbe(expecting: Cluster.Gossip.self) + self.pC = testKit.spawnTestProbe(expecting: Cluster.Gossip.self) + initializeLogics() + } + + private func initializeLogics() { + self.logicA = makeLogic(self.systemA, self.pA) + self.logicB = makeLogic(self.systemB, self.pB) + self.logicC = makeLogic(self.systemC, self.pC) + } + + private func makeLogic(_ system: ActorSystem, _ probe: ActorTestProbe) -> MembershipGossipLogic { + MembershipGossipLogic( + GossipLogicContext( + ownerContext: self.testKit(system).makeFakeContext(), + gossipIdentifier: StringGossipIdentifier("membership") + ), + notifyOnGossipRef: probe.ref + ) + } + + // ==== ------------------------------------------------------------------------------------------------------------ + // MARK: Tests + + func test_pickMostBehindNode() throws { + let gossip = Cluster.Gossip.parse( + """ + A.up B.joining C.up + A: A@5 B@5 C@6 + B: A@5 B@1 C@1 + C: A@5 B@5 C@6 + """, + owner: systemA.cluster.node, nodes: nodes + ) + logicA.localGossipUpdate(payload: gossip) + + + + let round1 = logicA.selectPeers(peers: self.peers(of: logicA)) + round1.shouldEqual([self.b]) + } + + func test_stopCondition_converged() throws { + let gossip = Cluster.Gossip.parse( + """ + A.up B.joining C.up + A: A@5 B@5 C@6 + B: A@5 B@5 C@6 + C: A@5 B@5 C@6 + """, + owner: systemA.cluster.node, nodes: nodes + ) + logicA.localGossipUpdate(payload: gossip) + + let round1 = logicA.selectPeers(peers: self.peers(of: logicA)) + round1.shouldBeEmpty() + } + + func test_avgRounds_untilConvergence() throws { + let simulations = 10 + var roundCounts: [Int] = [] + for _ in 1...simulations { + self.initializeLogics() + + var gossipA = Cluster.Gossip.parse( + """ + A.up B.up C.up + A: A@5 B@3 C@3 + B: A@3 B@3 C@3 + C: A@3 B@3 C@3 + """, + owner: systemA.cluster.node, nodes: nodes + ) + logicA.localGossipUpdate(payload: gossipA) + + var gossipB = Cluster.Gossip.parse( + """ + A.up B.joining C.joining + A: A@3 B@3 C@3 + B: A@3 B@3 C@3 + C: A@3 B@3 C@3 + """, + owner: systemB.cluster.node, nodes: nodes + ) + logicB.localGossipUpdate(payload: gossipB) + + var gossipC = Cluster.Gossip.parse( + """ + A.up B.joining C.joining + A: A@3 B@3 C@3 + B: A@3 B@3 C@3 + C: A@3 B@3 C@3 + """, + owner: systemC.cluster.node, nodes: nodes + ) + logicC.localGossipUpdate(payload: gossipC) + + func allConverged(gossips: [Cluster.Gossip]) -> Bool { + var allSatisfied = true // on purpose not via .allSatisfy since we want to print status of each logic + for g in gossips.sorted(by: { $0.owner.node.systemName < $1.owner.node.systemName }) { + let converged = g.converged() + let convergenceStatus = converged ? "(locally assumed) converged" : "not converged" + pinfo("\(g.owner.node.systemName): \(convergenceStatus)") + allSatisfied = allSatisfied && converged + } + return allSatisfied + } + + func simulateGossipRound() { + // we shuffle the gossips to simulate the slight timing differences -- not always will the "first" node be the first where the timers trigger + // and definitely not always will it always _remain_ the first to be gossiping; there may be others still gossiping around spreading their "not super complete" + // information. + let participatingGossips = logics.shuffled() + for logic in participatingGossips { + let selectedPeers: [AddressableActorRef] = logic.selectPeers(peers: self.peers(of: logic)) + pinfo("[\(logic.nodeName)] selected peers: \(selectedPeers.map({$0.address.node!.node.systemName}))") + + for targetPeer in selectedPeers { + let targetGossip = logic.makePayload(target: targetPeer) + if let gossip = targetGossip { + // pinfo(" \(logic.nodeName) -> \(targetPeer.address.node!.node.systemName): \(pretty: gossip)") + pinfo(" \(logic.nodeName) -> \(targetPeer.address.node!.node.systemName)") + + let targetLogic = selectLogic(targetPeer) + targetLogic.receiveGossip(origin: self.origin(logic), payload: gossip) + + pinfo("updated [\(targetPeer.address.node!.node.systemName)] gossip: \(targetLogic.latestGossip)") + switch targetPeer.address.node!.node.systemName { + case "A": gossipA = targetLogic.latestGossip + case "B": gossipB = targetLogic.latestGossip + case "C": gossipC = targetLogic.latestGossip + default: fatalError("No gossip storage space for \(targetPeer)") + } + } else { + () // skipping target... + } + } + } + } + + var rounds = 0 + pnote("~~~~~~~~~~~~ new gossip instance ~~~~~~~~~~~~") + while !allConverged(gossips: [gossipA, gossipB, gossipC]) { + rounds += 1 + pnote("Next gossip round (\(rounds))...") + simulateGossipRound() + } + + pnote("All peers converged after: [\(rounds) rounds]") + roundCounts += [rounds] + } + pnote("Finished [\(simulations)] simulations, rounds: \(roundCounts) (\(Double(roundCounts.reduce(0, +)) / Double(simulations)) avg)") + } + + // ==== ---------------------------------------------------------------------------------------------------------------- + // MARK: Support functions + + func peers(of logic: MembershipGossipLogic) -> [AddressableActorRef] { + Array(self.allPeers.filter { $0.address.node! != logic.localNode }) + } + + func selectLogic(_ peer: AddressableActorRef) -> MembershipGossipLogic { + guard let node = peer.address.node else { + fatalError("MUST have node, was: \(peer.address)") + } + + switch node.node.systemName { + case "A": return self.logicA + case "B": return self.logicB + case "C": return self.logicC + default: fatalError("No logic for peer: \(peer)") + } + } + + func origin(_ logic: MembershipGossipLogic) -> AddressableActorRef { + if ObjectIdentifier(logic) == ObjectIdentifier(logicA) { + return self.a + } else if ObjectIdentifier(logic) == ObjectIdentifier(logicB) { + return self.b + } else if ObjectIdentifier(logic) == ObjectIdentifier(logicC) { + return self.c + } else { + fatalError("No addressable peer for logic: \(logic)") + } + } +} + +extension MembershipGossipLogic { + var nodeName: String { + self.localNode.node.systemName + } +} \ No newline at end of file From 45c286f84e8e639a2fc60be6cceb0d227623f01d Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Mon, 22 Jun 2020 16:15:05 +0900 Subject: [PATCH 02/15] =membership improve peer selection --- .../Cluster+MembershipGossipLogic.swift | 69 +++++++++++++++++-- .../Cluster/MembershipGossipLogicTests.swift | 21 +++++- 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift index 7bfe5c040..358ee6ddc 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift @@ -26,6 +26,8 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { internal var latestGossip: Cluster.Gossip private let notifyOnGossipRef: ActorRef + private var gossipPeers: [AddressableActorRef] = [] + init(_ context: Context, notifyOnGossipRef: ActorRef) { self.context = context self.notifyOnGossipRef = notifyOnGossipRef @@ -36,22 +38,48 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { // MARK: Spreading gossip // TODO: implement better, only peers which are "behind" - func selectPeers(peers: [AddressableActorRef]) -> [AddressableActorRef] { + func selectPeers(peers _peers: [AddressableActorRef]) -> [AddressableActorRef] { // how many peers we select in each gossip round, // we could for example be dynamic and notice if we have 10+ nodes, we pick 2 members to speed up the dissemination etc. let n = 1 - var selectedPeers: [AddressableActorRef] = [] - selectedPeers.reserveCapacity(n) + self.updateActivePeers(peers: _peers) - for peer in peers.shuffled() - where selectedPeers.count < n && self.shouldGossipWith(peer) { - selectedPeers.append(peer) + var selectedPeers: [AddressableActorRef] = [] + selectedPeers.reserveCapacity(min(n, self.gossipPeers.count)) + + /// Trust the order of peers in gossipPeers for the selection; see `updateActivePeers` for logic of the ordering. + for peer in self.gossipPeers + where selectedPeers.count < n { + if self.shouldGossipWith(peer) { + selectedPeers.append(peer) + } } return selectedPeers } + private func updateActivePeers(peers: [AddressableActorRef]) { + if let changed = Self.peersChanged(known: self.gossipPeers, current: peers) { + if !changed.removed.isEmpty { + let removedPeers = Set(changed.removed) + self.gossipPeers = self.gossipPeers.filter { !removedPeers.contains($0) } + } + + for peer in changed.added { + // Newly added members are inserted at a random spot in the list of members + // to ping, to have a better distribution of messages to this node from all + // other nodes. If for example all nodes would add it to the end of the list, + // it would take a longer time until it would be pinged for the first time + // and also likely receive multiple pings within a very short time frame. + // + // This is adopted from the SWIM membership implementation and related papers. + let insertIndex = Int.random(in: self.gossipPeers.startIndex ... self.gossipPeers.endIndex) + self.gossipPeers.insert(peer, at: insertIndex) + } + } + } + func makePayload(target: AddressableActorRef) -> Cluster.Gossip? { // today we don't trim payloads at all self.latestGossip @@ -88,6 +116,35 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { // } } + // TODO may also want to return "these were removed" if we need to make any internal cleanup + static func peersChanged(known: [AddressableActorRef], current: [AddressableActorRef]) -> PeersChanged? { + // TODO: a bit lazy implementation + let knownSet = Set(known) + let currentSet = Set(current) + + let added = currentSet.subtracting(knownSet) + let removed = knownSet.subtracting(currentSet) + + if added.isEmpty && removed.isEmpty { + return nil + } else { + return PeersChanged( + added: added, + removed: removed + ) + } + } + struct PeersChanged { + let added: Set + let removed: Set + + init(added: Set, removed: Set) { + assert(!added.isEmpty || !removed.isEmpty, "PeersChanged yet both added/removed are empty!") + self.added = added + self.removed = removed + } + } + // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift index e66dd126b..127dc6e14 100644 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift @@ -140,7 +140,7 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { var roundCounts: [Int] = [] for _ in 1...simulations { self.initializeLogics() - + var gossipA = Cluster.Gossip.parse( """ A.up B.up C.up @@ -231,6 +231,25 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { pnote("Finished [\(simulations)] simulations, rounds: \(roundCounts) (\(Double(roundCounts.reduce(0, +)) / Double(simulations)) avg)") } + func test_logic_peersChanged() throws { + let all = [a, b, c] + let known: [AddressableActorRef] = [a] + let less: [AddressableActorRef] = [] + let more: [AddressableActorRef] = [a, b] + + let res1 = MembershipGossipLogic.peersChanged(known: known, current: less) + res1!.removed.shouldEqual([a]) + res1!.added.shouldEqual([]) + + let res2 = MembershipGossipLogic.peersChanged(known: known, current: more) + res2!.removed.shouldEqual([]) + res2!.added.shouldEqual([b]) + + let res3 = MembershipGossipLogic.peersChanged(known: [], current: all) + res3!.removed.shouldEqual([]) + res3!.added.shouldEqual([a, b, c]) + } + // ==== ---------------------------------------------------------------------------------------------------------------- // MARK: Support functions From f063c472566a668fee25653fd7f396e98b45193b Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Tue, 23 Jun 2020 01:28:20 +0900 Subject: [PATCH 03/15] Gossip now handles user provided ACKs, Membership peer selection improvements --- .../DistributedActors/CRDT/CRDT+Gossip.swift | 13 +++- .../CRDT/CRDT+ReplicatorShell.swift | 4 +- .../Cluster/Cluster+Gossip.swift | 5 ++ .../Cluster+MembershipGossipLogic.swift | 75 ++++++++++++------- .../Cluster/ClusterShell.swift | 6 +- .../Cluster/ClusterShellState.swift | 4 +- .../Gossip/Gossip+Logic.swift | 41 +++++----- .../Gossip/Gossip+Serialization.swift | 2 +- .../Gossip/Gossip+Shell.swift | 63 +++++++--------- .../Gossip/PeerSelection.swift | 4 +- .../Serialization/Serialization.swift | 6 +- .../Cluster/MembershipGossipLogicTests.swift | 15 ++-- 12 files changed, 136 insertions(+), 102 deletions(-) diff --git a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift index 27658f1b0..f17d41cdc 100644 --- a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift +++ b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift @@ -21,6 +21,7 @@ extension CRDT { /// about them (e.g. through direct replication). final class GossipReplicatorLogic: GossipLogic { typealias Envelope = CRDT.Gossip + typealias Acknowledgement = CRDT.GossipAck let identity: CRDT.Identity let context: Context @@ -79,7 +80,7 @@ extension CRDT { self.latest } - func receivePayloadACK(target: AddressableActorRef, confirmedDeliveryOf envelope: CRDT.Gossip) { + func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: CRDT.Gossip) { guard (self.latest.map { $0.payload.equalState(to: envelope.payload) } ?? false) else { // received an ack for something, however it's not the "latest" anymore, so we need to gossip to target anyway return @@ -87,13 +88,13 @@ extension CRDT { // TODO: in v3 this would translate to ACKing specific deltas for this target // good, the latest gossip is still the same as was confirmed here, so we can mark it acked - self.peersAckedOurLatestGossip.insert(target.address) + self.peersAckedOurLatestGossip.insert(peer.address) } // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - func receiveGossip(origin: AddressableActorRef, payload: CRDT.Gossip) { + func receiveGossip(origin: AddressableActorRef, payload: CRDT.Gossip) -> CRDT.GossipAck? { // merge the datatype locally, and update our information about the origin's knowledge about this datatype // (does it already know about our data/all-deltas-we-are-aware-of or not) self.mergeInbound(from: origin, payload) @@ -101,6 +102,8 @@ extension CRDT { // notify the direct replicator to update all local `actorOwned` CRDTs. // TODO: the direct replicator may choose to delay flushing this information a bit to avoid much data churn see `settings.crdt.` self.replicatorControl.tellGossipWrite(id: self.identity, data: payload.payload) + + return .init() // always ack } func localGossipUpdate(payload: CRDT.Gossip) { @@ -178,7 +181,7 @@ extension CRDT.Identity: GossipIdentifier { extension CRDT { /// The gossip to be spread about a specific CRDT (identity). struct Gossip: GossipEnvelopeProtocol, CustomStringConvertible, CustomPrettyStringConvertible { - struct Metadata: Codable {} + struct Metadata: Codable {} // FIXME: remove, seems we dont need metadata explicitly here typealias Payload = StateBasedCRDT var metadata: Metadata @@ -201,6 +204,8 @@ extension CRDT { "CRDT.Gossip(metadata: \(metadata), payload: \(payload))" } } + + struct GossipAck: Codable {} } extension CRDT.Gossip { diff --git a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift index 90c7ea2aa..de353572c 100644 --- a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift +++ b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift @@ -33,7 +33,7 @@ extension CRDT.Replicator { typealias RemoteDeleteResult = CRDT.Replicator.RemoteCommand.DeleteResult private let directReplicator: CRDT.Replicator.Instance - private var gossipReplication: GossipControl! + private var gossipReplication: GossipControl! // TODO: better name; this is the control from Gossip -> Local struct LocalControl { @@ -74,6 +74,8 @@ extension CRDT.Replicator { self.gossipReplication = try Gossiper.start( context, name: "gossip", + of: CRDT.Gossip.self, + ofAcknowledgement: CRDT.GossipAck.self, settings: Gossiper.Settings( gossipInterval: self.settings.gossipInterval, gossipIntervalRandomFactor: self.settings.gossipIntervalRandomFactor, diff --git a/Sources/DistributedActors/Cluster/Cluster+Gossip.swift b/Sources/DistributedActors/Cluster/Cluster+Gossip.swift index 08964a01f..bc9b1b356 100644 --- a/Sources/DistributedActors/Cluster/Cluster+Gossip.swift +++ b/Sources/DistributedActors/Cluster/Cluster+Gossip.swift @@ -167,6 +167,11 @@ extension Cluster { return !laggingBehindMemberFound } } + +// struct GossipAck: Codable { +// let owner: UniqueNode +// var seen: Cluster.Gossip.SeenTable +// } } extension Cluster.Gossip: GossipEnvelopeProtocol { diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift index 358ee6ddc..6c56384ac 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift @@ -19,6 +19,7 @@ import NIO final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { typealias Envelope = Cluster.Gossip + typealias Acknowledgement = Cluster.Gossip // TODO: GossipAck instead, a more minimal one; just the peers status private let context: Context internal lazy var localNode: UniqueNode = self.context.system.cluster.node @@ -26,7 +27,12 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { internal var latestGossip: Cluster.Gossip private let notifyOnGossipRef: ActorRef - private var gossipPeers: [AddressableActorRef] = [] + /// We store and use a shuffled yet stable order for gossiping peers. + /// See `updateActivePeers` for details. + private var peers: [AddressableActorRef] = [] + + /// See `updateActivePeers` and `receiveGossip` for details. + private var peerLastSeenGossip: [AddressableActorRef: Cluster.Gossip] = [:] init(_ context: Context, notifyOnGossipRef: ActorRef) { self.context = context @@ -44,13 +50,11 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { let n = 1 self.updateActivePeers(peers: _peers) - var selectedPeers: [AddressableActorRef] = [] - selectedPeers.reserveCapacity(min(n, self.gossipPeers.count)) + selectedPeers.reserveCapacity(min(n, self.peers.count)) /// Trust the order of peers in gossipPeers for the selection; see `updateActivePeers` for logic of the ordering. - for peer in self.gossipPeers - where selectedPeers.count < n { + for peer in self.peers where selectedPeers.count < n { if self.shouldGossipWith(peer) { selectedPeers.append(peer) } @@ -60,10 +64,16 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { } private func updateActivePeers(peers: [AddressableActorRef]) { - if let changed = Self.peersChanged(known: self.gossipPeers, current: peers) { + if let changed = Self.peersChanged(known: self.peers, current: peers) { + // 1) remove any peers which are no longer active + // - from the peers list + // - from their gossip storage, we'll never gossip with them again after all if !changed.removed.isEmpty { let removedPeers = Set(changed.removed) - self.gossipPeers = self.gossipPeers.filter { !removedPeers.contains($0) } + self.peers = self.peers.filter { !removedPeers.contains($0) } + changed.removed.forEach { removed in + _ = self.peerLastSeenGossip.removeValue(forKey: removed) + } } for peer in changed.added { @@ -74,18 +84,20 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { // and also likely receive multiple pings within a very short time frame. // // This is adopted from the SWIM membership implementation and related papers. - let insertIndex = Int.random(in: self.gossipPeers.startIndex ... self.gossipPeers.endIndex) - self.gossipPeers.insert(peer, at: insertIndex) + let insertIndex = Int.random(in: self.peers.startIndex ... self.peers.endIndex) + self.peers.insert(peer, at: insertIndex) } } } func makePayload(target: AddressableActorRef) -> Cluster.Gossip? { // today we don't trim payloads at all + // TODO: trim some information? self.latestGossip } - func receivePayloadACK(target: AddressableActorRef, confirmedDeliveryOf envelope: Cluster.Gossip) { + func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Cluster.Gossip) { + // TODO: forward what we know about the from `peer` // nothing to do } @@ -96,27 +108,33 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { return false } -// guard let remoteSeenVersion = self.latestGossip.seen.version(at: remoteNode) else { - guard self.latestGossip.seen.version(at: remoteNode) != nil else { + guard let peerSeenVersion = self.peerLastSeenGossip[peer]?.seen.version(at: remoteNode) else { // this peer has never seen any information from us, so we definitely want to push a gossip return true } - // FIXME: this is longer than may be necessary, optimize some more - return true +// // FIXME: this is longer than may be necessary, optimize some more +// return true // TODO: optimize some more; but today we need to keep gossiping until all VVs are the same, because convergence depends on this -// switch self.latestGossip.seen.compareVersion(observedOn: self.localNode, to: remoteSeenVersion) { -// case .happenedBefore, .same: -// // we have strictly less information than the peer, no need to gossip to it -// return false -// case .concurrent, .happenedAfter: -// // we have strictly concurrent or more information the peer, gossip with it -// return true -// } + switch self.latestGossip.seen.compareVersion(observedOn: self.localNode, to: peerSeenVersion) { + case .happenedBefore: + pprint("[\(self.localNode.node.systemName)] happenedBefore [\(remoteNode.node.systemName)]") + return false + case .same: + pprint("[\(self.localNode.node.systemName)] same [\(remoteNode.node.systemName)]") + return false + case .concurrent: + // we have strictly concurrent or more information the peer, gossip with it + pprint("[\(self.localNode.node.systemName)] concurrent [\(remoteNode.node.systemName)]") + return true + case .happenedAfter: + pprint("[\(self.localNode.node.systemName)] happenedAfter [\(remoteNode.node.systemName)] GOSSIP") + return true + } } - // TODO may also want to return "these were removed" if we need to make any internal cleanup + // TODO: may also want to return "these were removed" if we need to make any internal cleanup static func peersChanged(known: [AddressableActorRef], current: [AddressableActorRef]) -> PeersChanged? { // TODO: a bit lazy implementation let knownSet = Set(known) @@ -125,7 +143,7 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { let added = currentSet.subtracting(knownSet) let removed = knownSet.subtracting(currentSet) - if added.isEmpty && removed.isEmpty { + if added.isEmpty, removed.isEmpty { return nil } else { return PeersChanged( @@ -134,6 +152,7 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { ) } } + struct PeersChanged { let added: Set let removed: Set @@ -148,9 +167,13 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - func receiveGossip(origin: AddressableActorRef, payload: Cluster.Gossip) { + func receiveGossip(origin: AddressableActorRef, payload: Cluster.Gossip) -> Cluster.Gossip? { + self.peerLastSeenGossip[origin] = payload self.mergeInbound(payload) self.notifyOnGossipRef.tell(self.latestGossip) + + // FIXME: optimized reply here; consider what exactly we should reply with + return self.latestGossip } func localGossipUpdate(payload: Cluster.Gossip) { @@ -185,7 +208,7 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { } var description: String { - "MembershipGossipLogic(\(localNode))" + "MembershipGossipLogic(\(self.localNode))" } } diff --git a/Sources/DistributedActors/Cluster/ClusterShell.swift b/Sources/DistributedActors/Cluster/ClusterShell.swift index a97091a41..cba2915d5 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell.swift @@ -423,7 +423,7 @@ extension ClusterShell { context.log.info("Bound to \(chan.localAddress.map { $0.description } ?? "")") // TODO: Membership.Gossip? - let gossipControl: GossipControl = try Gossiper.start( + let gossipControl: GossipControl = try Gossiper.start( context, name: "\(ActorPath._clusterGossip.name)", props: ._wellKnown, @@ -431,7 +431,7 @@ extension ClusterShell { gossipInterval: clusterSettings.membershipGossipInterval, gossipIntervalRandomFactor: clusterSettings.membershipGossipIntervalRandomFactor, peerDiscovery: .onClusterMember(atLeast: .joining, resolve: { member in - let resolveContext = ResolveContext.Message>(address: ._clusterGossip(on: member.node), system: context.system) + let resolveContext = ResolveContext.Message>(address: ._clusterGossip(on: member.node), system: context.system) return context.system._resolve(context: resolveContext).asAddressable() }) ), @@ -636,7 +636,7 @@ extension ClusterShell { // TODO: make it cleaner? though we decided to go with manual peer management as the ClusterShell owns it, hm // TODO: consider receptionist instead of this; we're "early" but receptionist could already be spreading its info to this node, since we associated. - let gossipPeer: GossipShell.Ref = context.system._resolve( + let gossipPeer: GossipShell.Ref = context.system._resolve( context: .init(address: ._clusterGossip(on: change.member.node), system: context.system) ) // FIXME: make sure that if the peer terminated, we don't add it again in here, receptionist would be better then to power this... diff --git a/Sources/DistributedActors/Cluster/ClusterShellState.swift b/Sources/DistributedActors/Cluster/ClusterShellState.swift index 6db8de998..94e5ebbaa 100644 --- a/Sources/DistributedActors/Cluster/ClusterShellState.swift +++ b/Sources/DistributedActors/Cluster/ClusterShellState.swift @@ -60,7 +60,7 @@ internal struct ClusterShellState: ReadOnlyClusterState { internal var _handshakes: [Node: HandshakeStateMachine.State] = [:] - let gossipControl: GossipControl + let gossipControl: GossipControl /// Updating the `latestGossip` causes the gossiper to be informed about it, such that the next time it does a gossip round /// it uses the latest gossip available. @@ -101,7 +101,7 @@ internal struct ClusterShellState: ReadOnlyClusterState { settings: ClusterSettings, channel: Channel, events: EventStream, - gossipControl: GossipControl, + gossipControl: GossipControl, log: Logger ) { self.log = log diff --git a/Sources/DistributedActors/Gossip/Gossip+Logic.swift b/Sources/DistributedActors/Gossip/Gossip+Logic.swift index 0acc97085..0bd634ed0 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Logic.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Logic.swift @@ -39,7 +39,9 @@ import Logging /// - SeeAlso: `Cluster.Gossip` for the Actor System's own gossip mechanism for membership dissemination public protocol GossipLogic { associatedtype Envelope: GossipEnvelopeProtocol - typealias Context = GossipLogicContext + associatedtype Acknowledgement: Codable + + typealias Context = GossipLogicContext // init(context: Context) // TODO: a form of context? @@ -62,15 +64,16 @@ public protocol GossipLogic { /// Eg. if gossip is sent to 2 peers, it is NOT deterministic which of the acks returns first (or at all!). /// /// - Parameters: - /// - target: The target which has acknowlaged the gossiped payload. + /// - peer: The target which has acknowledged the gossiped payload. /// It corresponds to the parameter that was passed to the `makePayload(target:)` which created this gossip payload. + /// - acknowledgement: acknowledgement sent by the peer /// - envelope: - mutating func receivePayloadACK(target: AddressableActorRef, confirmedDeliveryOf envelope: Envelope) + mutating func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - mutating func receiveGossip(origin: AddressableActorRef, payload: Envelope) + mutating func receiveGossip(origin: AddressableActorRef, payload: Envelope) -> Acknowledgement? mutating func localGossipUpdate(payload: Envelope) @@ -85,12 +88,12 @@ extension GossipLogic { } } -public struct GossipLogicContext { +public struct GossipLogicContext { public let gossipIdentifier: GossipIdentifier - private let ownerContext: ActorContext.Message> + private let ownerContext: ActorContext.Message> - internal init(ownerContext: ActorContext.Message>, gossipIdentifier: GossipIdentifier) { + internal init(ownerContext: ActorContext.Message>, gossipIdentifier: GossipIdentifier) { self.ownerContext = ownerContext self.gossipIdentifier = gossipIdentifier } @@ -114,29 +117,29 @@ public struct GossipLogicContext { } } -public struct AnyGossipLogic: GossipLogic, CustomStringConvertible { +public struct AnyGossipLogic: GossipLogic, CustomStringConvertible { @usableFromInline let _selectPeers: ([AddressableActorRef]) -> [AddressableActorRef] @usableFromInline let _makePayload: (AddressableActorRef) -> Envelope? @usableFromInline - let _receivePayloadACK: (AddressableActorRef, Envelope) -> Void + let _receiveGossip: (AddressableActorRef, Envelope) -> Acknowledgement? @usableFromInline - let _receiveGossip: (AddressableActorRef, Envelope) -> Void + let _receiveAcknowledgement: (AddressableActorRef, Acknowledgement, Envelope) -> Void + @usableFromInline let _localGossipUpdate: (Envelope) -> Void - @usableFromInline let _receiveSideChannelMessage: (Any) throws -> Void public init(_ logic: Logic) - where Logic: GossipLogic, Logic.Envelope == Envelope { + where Logic: GossipLogic, Logic.Envelope == Envelope, Logic.Acknowledgement == Acknowledgement { var l = logic self._selectPeers = { l.selectPeers(peers: $0) } self._makePayload = { l.makePayload(target: $0) } - self._receivePayloadACK = { l.receivePayloadACK(target: $0, confirmedDeliveryOf: $1) } - self._receiveGossip = { l.receiveGossip(origin: $0, payload: $1) } + + self._receiveAcknowledgement = { l.receiveAcknowledgement(from: $0, acknowledgement: $1, confirmsDeliveryOf: $2) } self._localGossipUpdate = { l.localGossipUpdate(payload: $0) } self._receiveSideChannelMessage = { try l.receiveSideChannelMessage(message: $0) } @@ -150,12 +153,12 @@ public struct AnyGossipLogic: GossipLogic, Cus self._makePayload(target) } - public func receivePayloadACK(target: AddressableActorRef, confirmedDeliveryOf envelope: Envelope) { - self._receivePayloadACK(target, envelope) + public func receiveGossip(origin: AddressableActorRef, payload: Envelope) -> Acknowledgement? { + self._receiveGossip(origin, payload) } - public func receiveGossip(origin: AddressableActorRef, payload: Envelope) { - self._receiveGossip(origin, payload) + public func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) { + self._receiveAcknowledgement(peer, acknowledgement, envelope) } public func localGossipUpdate(payload: Envelope) { @@ -183,5 +186,3 @@ public protocol GossipEnvelopeProtocol: Codable { var metadata: Metadata { get } var payload: Payload { get } } - -public struct GossipACK: Codable {} // TODO: Make Acknowlagement an associated type on GossipEnvelopeProtocol! diff --git a/Sources/DistributedActors/Gossip/Gossip+Serialization.swift b/Sources/DistributedActors/Gossip/Gossip+Serialization.swift index 912144a3b..19feabff9 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Serialization.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Serialization.swift @@ -54,7 +54,7 @@ extension GossipShell.Message: Codable { let payload = try context.serialization.deserialize(as: Envelope.self, from: .data(payloadPayload), using: payloadManifest) let ackRefAddress = try container.decode(ActorAddress.self, forKey: .ackRef) - let ackRef = context.resolveActorRef(GossipACK.self, identifiedBy: ackRefAddress) + let ackRef = context.resolveActorRef(Acknowledgement.self, identifiedBy: ackRefAddress) self = .gossip(identity: identifier, origin: origin, payload, ackRef: ackRef) } diff --git a/Sources/DistributedActors/Gossip/Gossip+Shell.swift b/Sources/DistributedActors/Gossip/Gossip+Shell.swift index 68525dbb1..dc5844ba6 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Shell.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Shell.swift @@ -17,15 +17,15 @@ import Logging private let gossipTickKey: TimerKey = "gossip-tick" /// Convergent gossip is a gossip mechanism which aims to equalize some state across all peers participating. -internal final class GossipShell { +internal final class GossipShell { typealias Ref = ActorRef let settings: Gossiper.Settings - private let makeLogic: (ActorContext, GossipIdentifier) -> AnyGossipLogic + private let makeLogic: (ActorContext, GossipIdentifier) -> AnyGossipLogic /// Payloads to be gossiped on gossip rounds - private var gossipLogics: [AnyGossipIdentifier: AnyGossipLogic] + private var gossipLogics: [AnyGossipIdentifier: AnyGossipLogic] typealias PeerRef = ActorRef private var peers: Set @@ -33,7 +33,7 @@ internal final class GossipShell { fileprivate init( settings: Gossiper.Settings, makeLogic: @escaping (Logic.Context) -> Logic - ) where Logic: GossipLogic, Logic.Envelope == Envelope { + ) where Logic: GossipLogic, Logic.Envelope == Envelope, Logic.Acknowledgement == Acknowledgement { self.settings = settings self.makeLogic = { shellContext, id in let logicContext = GossipLogicContext(ownerContext: shellContext, gossipIdentifier: id) @@ -91,7 +91,7 @@ internal final class GossipShell { identifier: GossipIdentifier, origin: ActorRef, payload: Envelope, - ackRef: ActorRef + ackRef: ActorRef ) { context.log.trace("Received gossip [\(identifier.gossipIdentifier)]", metadata: [ "gossip/identity": "\(identifier.gossipIdentifier)", @@ -99,13 +99,11 @@ internal final class GossipShell { "gossip/incoming": Logger.MetadataValue.pretty(payload), ]) - // TODO: we could handle some actions if it issued some - let logic: AnyGossipLogic = self.getEnsureLogic(context, identifier: identifier) - - // TODO: we could handle directives from the logic - logic.receiveGossip(origin: origin.asAddressable(), payload: payload) + let logic = self.getEnsureLogic(context, identifier: identifier) - ackRef.tell(.init()) // TODO: allow the user to return an ACK from receiveGossip + if let ack = logic.receiveGossip(origin: origin.asAddressable(), payload: payload) { + ackRef.tell(ack) + } } private func onLocalPayloadUpdate( @@ -125,8 +123,8 @@ internal final class GossipShell { // TODO: bump local version vector; once it is in the envelope } - private func getEnsureLogic(_ context: ActorContext, identifier: GossipIdentifier) -> AnyGossipLogic { - let logic: AnyGossipLogic + private func getEnsureLogic(_ context: ActorContext, identifier: GossipIdentifier) -> AnyGossipLogic { + let logic: AnyGossipLogic if let existing = self.gossipLogics[identifier.asAnyGossipIdentifier] { logic = existing } else { @@ -187,15 +185,8 @@ internal final class GossipShell { continue } -// pprint(""" -// [\(context.system.cluster.node)] \ -// Selected [\(selectedPeers.count)] peers, \ -// from [\(allPeers.count)] peers: \(selectedPeers)\ -// PAYLOAD: \(pretty: payload) -// """) - - self.sendGossip(context, identifier: identifier, payload, to: selectedRef, onAck: { - logic.receivePayloadACK(target: selectedPeer, confirmedDeliveryOf: payload) + self.sendGossip(context, identifier: identifier, payload, to: selectedRef, onGossipAck: { ack in + logic.receiveAcknowledgement(from: selectedPeer, acknowledgement: ack, confirmsDeliveryOf: payload) }) } @@ -209,7 +200,7 @@ internal final class GossipShell { identifier: AnyGossipIdentifier, _ payload: Envelope, to target: PeerRef, - onAck: @escaping () -> Void + onGossipAck: @escaping (Acknowledgement) -> Void ) { context.log.trace("Sending gossip to \(target.address)", metadata: [ "gossip/target": "\(target.address)", @@ -217,7 +208,8 @@ internal final class GossipShell { "actor/message": Logger.MetadataValue.pretty(payload), ]) - let ack = target.ask(for: GossipACK.self, timeout: .seconds(3)) { replyTo in + // TODO: configurable timeout? + let ack = target.ask(for: Acknowledgement.self, timeout: .seconds(3)) { replyTo in Message.gossip(identity: identifier.underlying, origin: context.myself, payload, ackRef: replyTo) } @@ -227,7 +219,7 @@ internal final class GossipShell { context.log.trace("Gossip ACKed", metadata: [ "gossip/ack": "\(ack)", ]) - onAck() + onGossipAck(ack) case .failure: context.log.warning("Failed to ACK delivery [\(identifier.gossipIdentifier)] gossip \(payload) to \(target)") } @@ -361,7 +353,7 @@ extension GossipShell { extension GossipShell { enum Message { // gossip - case gossip(identity: GossipIdentifier, origin: ActorRef, Envelope, ackRef: ActorRef) + case gossip(identity: GossipIdentifier, origin: ActorRef, Envelope, ackRef: ActorRef) // local messages case updatePayload(identifier: GossipIdentifier, Envelope) @@ -381,20 +373,21 @@ extension GossipShell { /// A Gossiper public enum Gossiper { /// Spawns a gossip actor, that will periodically gossip with its peers about the provided payload. - static func start( + static func start( _ context: ActorRefFactory, name naming: ActorNaming, of type: Envelope.Type = Envelope.self, + ofAcknowledgement acknowledgementType: Acknowledgement.Type = Acknowledgement.self, props: Props = .init(), settings: Settings = .init(), makeLogic: @escaping (Logic.Context) -> Logic - ) throws -> GossipControl - where Logic: GossipLogic, Logic.Envelope == Envelope { + ) throws -> GossipControl + where Logic: GossipLogic, Logic.Envelope == Envelope, Logic.Acknowledgement == Acknowledgement { let ref = try context.spawn( naming, - of: GossipShell.Message.self, + of: GossipShell.Message.self, props: props, file: #file, line: #line, - GossipShell(settings: settings, makeLogic: makeLogic).behavior + GossipShell(settings: settings, makeLogic: makeLogic).behavior ) return GossipControl(ref) } @@ -403,15 +396,15 @@ public enum Gossiper { // ==== ---------------------------------------------------------------------------------------------------------------- // MARK: GossipControl -internal struct GossipControl { - private let ref: GossipShell.Ref +internal struct GossipControl { + private let ref: GossipShell.Ref - init(_ ref: GossipShell.Ref) { + init(_ ref: GossipShell.Ref) { self.ref = ref } /// Introduce a peer to the gossip group - func introduce(peer: GossipShell.Ref) { + func introduce(peer: GossipShell.Ref) { self.ref.tell(.introducePeer(peer)) } diff --git a/Sources/DistributedActors/Gossip/PeerSelection.swift b/Sources/DistributedActors/Gossip/PeerSelection.swift index e8acf1ac2..d40fc2847 100644 --- a/Sources/DistributedActors/Gossip/PeerSelection.swift +++ b/Sources/DistributedActors/Gossip/PeerSelection.swift @@ -28,7 +28,7 @@ public protocol PeerSelection { func select() -> Peers } -//public struct StableRandomRoundRobin { +// public struct StableRandomRoundRobin { // // var peerSet: Set // var peers: [Peer] @@ -60,4 +60,4 @@ public protocol PeerSelection { // return selectedPeers // } // -//} +// } diff --git a/Sources/DistributedActors/Serialization/Serialization.swift b/Sources/DistributedActors/Serialization/Serialization.swift index 4cb3126e0..dc61ba47a 100644 --- a/Sources/DistributedActors/Serialization/Serialization.swift +++ b/Sources/DistributedActors/Serialization/Serialization.swift @@ -116,7 +116,7 @@ public class Serialization { settings.register(ClusterShell.Message.self) settings.register(Cluster.Event.self) settings.register(Cluster.Gossip.self) - settings.register(GossipShell.Message.self) + settings.register(GossipShell.Message.self) settings.register(StringGossipIdentifier.self) // receptionist needs some special casing @@ -166,8 +166,8 @@ public class Serialization { settings.register(CRDT.GCounterDelta.self, serializerID: Serialization.ReservedID.CRDTGCounterDelta) // crdt gossip - settings.register(GossipACK.self) - settings.register(GossipShell.Message.self) // TODO: remove this, workaround since we ust strings rather than mangled names today + settings.register(CRDT.GossipAck.self) + settings.register(GossipShell.Message.self) settings.register(CRDT.Gossip.self) settings.register(CRDT.Gossip.Metadata.self) diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift index 127dc6e14..0c66c9d9e 100644 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift @@ -69,8 +69,8 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { self.systems = [systemA, systemB, systemC] self.nodes = systems.map { $0.cluster.node } - self.allPeers = try! systems.map { system -> ActorRef.Message> in - let ref: ActorRef.Message> = try system.spawn("peer", .receiveMessage { _ in .same }) + self.allPeers = try! systems.map { system -> ActorRef.Message> in + let ref: ActorRef.Message> = try system.spawn("peer", .receiveMessage { _ in .same }) return self.systemA._resolveKnownRemote(ref, onRemoteSystem: system) }.map { $0.asAddressable() } @@ -90,7 +90,7 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { private func makeLogic(_ system: ActorSystem, _ probe: ActorTestProbe) -> MembershipGossipLogic { MembershipGossipLogic( - GossipLogicContext( + GossipLogicContext( ownerContext: self.testKit(system).makeFakeContext(), gossipIdentifier: StringGossipIdentifier("membership") ), @@ -138,7 +138,8 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { func test_avgRounds_untilConvergence() throws { let simulations = 10 var roundCounts: [Int] = [] - for _ in 1...simulations { + var messageCounts: [Int] = Array(repeating: 0, count: simulations) + for simulationNr in 1...simulations { self.initializeLogics() var gossipA = Cluster.Gossip.parse( @@ -195,6 +196,8 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { pinfo("[\(logic.nodeName)] selected peers: \(selectedPeers.map({$0.address.node!.node.systemName}))") for targetPeer in selectedPeers { + messageCounts[simulationNr - 1] += 1 + let targetGossip = logic.makePayload(target: targetPeer) if let gossip = targetGossip { // pinfo(" \(logic.nodeName) -> \(targetPeer.address.node!.node.systemName): \(pretty: gossip)") @@ -228,7 +231,9 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { pnote("All peers converged after: [\(rounds) rounds]") roundCounts += [rounds] } - pnote("Finished [\(simulations)] simulations, rounds: \(roundCounts) (\(Double(roundCounts.reduce(0, +)) / Double(simulations)) avg)") + pnote("Finished [\(simulations)] simulationsRounds: \(roundCounts)") + pnote(" Rounds: \(roundCounts) (\(Double(roundCounts.reduce(0, +)) / Double(simulations)) avg)") + pnote(" Messages: \(messageCounts) (\(Double(messageCounts.reduce(0, +)) / Double(simulations)) avg)") } func test_logic_peersChanged() throws { From 44203c4cff08d3485a6b48a4b630e45831e1e65b Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Tue, 23 Jun 2020 16:34:05 +0900 Subject: [PATCH 04/15] =gossip,cluster make membership stop condition test a simulation test --- .../DistributedActors/CRDT/CRDT+Gossip.swift | 12 +- .../Cluster+MembershipGossipLogic.swift | 68 ++-- .../Gossip/Gossip+Logic.swift | 18 +- .../Gossip/Gossip+Shell.swift | 4 +- ...MembershipGossipLogicSimulationTests.swift | 327 ++++++++++++++++++ .../Cluster/MembershipGossipLogicTests.swift | 142 ++------ 6 files changed, 402 insertions(+), 169 deletions(-) create mode 100644 Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift diff --git a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift index f17d41cdc..538608523 100644 --- a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift +++ b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift @@ -94,20 +94,20 @@ extension CRDT { // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - func receiveGossip(origin: AddressableActorRef, payload: CRDT.Gossip) -> CRDT.GossipAck? { + func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> CRDT.GossipAck? { // merge the datatype locally, and update our information about the origin's knowledge about this datatype // (does it already know about our data/all-deltas-we-are-aware-of or not) - self.mergeInbound(from: origin, payload) + self.mergeInbound(from: peer, gossip) // notify the direct replicator to update all local `actorOwned` CRDTs. // TODO: the direct replicator may choose to delay flushing this information a bit to avoid much data churn see `settings.crdt.` - self.replicatorControl.tellGossipWrite(id: self.identity, data: payload.payload) + self.replicatorControl.tellGossipWrite(id: self.identity, data: gossip.payload) - return .init() // always ack + return CRDT.GossipAck() } - func localGossipUpdate(payload: CRDT.Gossip) { - self.mergeInbound(from: nil, payload) + func localGossipUpdate(gossip: CRDT.Gossip) { + self.mergeInbound(from: nil, gossip) // during the next gossip round we'll gossip the latest most-up-to date version now; // no need to schedule that, we'll be called when it's time. } diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift index 6c56384ac..ca414b2f9 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift @@ -31,8 +31,11 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { /// See `updateActivePeers` for details. private var peers: [AddressableActorRef] = [] + /// During 1:1 gossip interactions, update this table, which means "we definitely know the specific node has seen our version VV at ..." + /// /// See `updateActivePeers` and `receiveGossip` for details. - private var peerLastSeenGossip: [AddressableActorRef: Cluster.Gossip] = [:] + // TODO: This can be optimized and it's enough if we keep a digest of the gossips; this way ACKs can just send the digest as well saving space. + private var lastGossipFrom: [AddressableActorRef: Cluster.Gossip] = [:] init(_ context: Context, notifyOnGossipRef: ActorRef) { self.context = context @@ -71,8 +74,8 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { if !changed.removed.isEmpty { let removedPeers = Set(changed.removed) self.peers = self.peers.filter { !removedPeers.contains($0) } - changed.removed.forEach { removed in - _ = self.peerLastSeenGossip.removeValue(forKey: removed) + changed.removed.forEach { removedPeer in + _ = self.lastGossipFrom.removeValue(forKey: removedPeer) } } @@ -97,8 +100,11 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { } func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Cluster.Gossip) { - // TODO: forward what we know about the from `peer` - // nothing to do + // 1) store the direct gossip we got from this peer; we can use this to know if there's no need to gossip to that peer by inspecting seen table equality + self.lastGossipFrom[peer] = acknowledgement + + // 2) use this to move forward the gossip as well + self.mergeInbound(gossip: acknowledgement) } /// True if the peers is "behind" in terms of information it has "seen" (as determined by comparing our and its seen tables). @@ -108,30 +114,16 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { return false } - guard let peerSeenVersion = self.peerLastSeenGossip[peer]?.seen.version(at: remoteNode) else { - // this peer has never seen any information from us, so we definitely want to push a gossip + guard let lastSeenGossipFromPeer = self.lastGossipFrom[peer] else { + // it's a peer we have not gotten any gossip from yet return true } -// // FIXME: this is longer than may be necessary, optimize some more -// return true +// pprint("\(self.localNode.node.systemName) = self.latestGossip.version = \(pretty: self.latestGossip)") +// pprint("\(self.localNode.node.systemName) = lastSeenGossipFromPeer[\(peer.address.node!.node.systemName)] = \(pretty: lastSeenGossipFromPeer)") - // TODO: optimize some more; but today we need to keep gossiping until all VVs are the same, because convergence depends on this - switch self.latestGossip.seen.compareVersion(observedOn: self.localNode, to: peerSeenVersion) { - case .happenedBefore: - pprint("[\(self.localNode.node.systemName)] happenedBefore [\(remoteNode.node.systemName)]") - return false - case .same: - pprint("[\(self.localNode.node.systemName)] same [\(remoteNode.node.systemName)]") - return false - case .concurrent: - // we have strictly concurrent or more information the peer, gossip with it - pprint("[\(self.localNode.node.systemName)] concurrent [\(remoteNode.node.systemName)]") - return true - case .happenedAfter: - pprint("[\(self.localNode.node.systemName)] happenedAfter [\(remoteNode.node.systemName)] GOSSIP") - return true - } + // TODO: can be replaced by a digest comparison + return self.latestGossip.seen != lastSeenGossipFromPeer.seen } // TODO: may also want to return "these were removed" if we need to make any internal cleanup @@ -167,17 +159,23 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - func receiveGossip(origin: AddressableActorRef, payload: Cluster.Gossip) -> Cluster.Gossip? { - self.peerLastSeenGossip[origin] = payload - self.mergeInbound(payload) + func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? { + // 1) mark that from that specific peer, we know it observed at least that version + self.lastGossipFrom[peer] = gossip + + // 2) move forward the gossip we store + self.mergeInbound(gossip: gossip) + + // 3) notify listeners self.notifyOnGossipRef.tell(self.latestGossip) - // FIXME: optimized reply here; consider what exactly we should reply with + // FIXME: optimize ack reply; this can contain only rows of seen tables where we are "ahead" (and always "our" row) + // no need to send back the entire tables if it's the same up to date ones as we just received return self.latestGossip } - func localGossipUpdate(payload: Cluster.Gossip) { - self.mergeInbound(payload) + func localGossipUpdate(gossip: Cluster.Gossip) { + self.mergeInbound(gossip: gossip) } // ==== ------------------------------------------------------------------------------------------------------------ @@ -194,16 +192,16 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { } switch sideChannelMessage { - case .localUpdate(let payload): - self.mergeInbound(payload) + case .localUpdate(let gossip): + self.mergeInbound(gossip: gossip) } } // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Utilities - private func mergeInbound(_ incoming: Cluster.Gossip) { - _ = self.latestGossip.mergeForward(incoming: incoming) + private func mergeInbound(gossip: Cluster.Gossip) { + _ = self.latestGossip.mergeForward(incoming: gossip) // effects are signalled via the ClusterShell, not here (it will also perform a merge) // TODO: a bit duplicated, could we maintain it here? } diff --git a/Sources/DistributedActors/Gossip/Gossip+Logic.swift b/Sources/DistributedActors/Gossip/Gossip+Logic.swift index 0bd634ed0..a5858458f 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Logic.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Logic.swift @@ -73,9 +73,9 @@ public protocol GossipLogic { // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - mutating func receiveGossip(origin: AddressableActorRef, payload: Envelope) -> Acknowledgement? + mutating func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? - mutating func localGossipUpdate(payload: Envelope) + mutating func localGossipUpdate(gossip: Envelope) /// Extra side channel, allowing for arbitrary outside interactions with this gossip logic. // TODO: We could consider making it typed perhaps... @@ -123,7 +123,7 @@ public struct AnyGossipLogic Envelope? @usableFromInline - let _receiveGossip: (AddressableActorRef, Envelope) -> Acknowledgement? + let _receiveGossip: (Envelope, AddressableActorRef) -> Acknowledgement? @usableFromInline let _receiveAcknowledgement: (AddressableActorRef, Acknowledgement, Envelope) -> Void @@ -137,10 +137,10 @@ public struct AnyGossipLogic Acknowledgement? { - self._receiveGossip(origin, payload) + public func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? { + self._receiveGossip(gossip, peer) } public func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) { self._receiveAcknowledgement(peer, acknowledgement, envelope) } - public func localGossipUpdate(payload: Envelope) { - self._localGossipUpdate(payload) + public func localGossipUpdate(gossip: Envelope) { + self._localGossipUpdate(gossip) } public func receiveSideChannelMessage(_ message: Any) throws { diff --git a/Sources/DistributedActors/Gossip/Gossip+Shell.swift b/Sources/DistributedActors/Gossip/Gossip+Shell.swift index dc5844ba6..7515d372c 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Shell.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Shell.swift @@ -101,7 +101,7 @@ internal final class GossipShell ActorSystem { + self.systems.first(where: { $0.name == id })! + } + + var nodes: [UniqueNode] { + self._nodes.map { $0.cluster.node } + } + + var mockPeers: [AddressableActorRef] = [] + + var testKit: ActorTestKit { + self.testKit(self.systems.first!) + } + + var logics: [MembershipGossipLogic] = [] + + func logic(_ id: String) -> MembershipGossipLogic { + guard let logic = (self.logics.first { $0.localNode.node.systemName == id }) else { + fatalError("No such logic for id: \(id)") + } + + return logic + } + + var gossips: [Cluster.Gossip] { + self.logics.map { $0.latestGossip } + } + + private func makeLogic(_ system: ActorSystem, _ probe: ActorTestProbe) -> MembershipGossipLogic { + MembershipGossipLogic( + GossipLogicContext( + ownerContext: self.testKit(system).makeFakeContext(), + gossipIdentifier: StringGossipIdentifier("membership") + ), + notifyOnGossipRef: probe.ref + ) + } + + // ==== ---------------------------------------------------------------------------------------------------------------- + // MARK: Simulation Tests + + func test_avgRounds_untilConvergence() throws { + let systemA = self.setUpNode("A") { settings in + settings.cluster.enabled = true + } + let systemB = self.setUpNode("B") + let systemC = self.setUpNode("C") + + let initialGossipState = + """ + A.up B.joining C.joining + A: A@3 B@3 C@3 + B: A@3 B@3 C@3 + C: A@3 B@3 C@3 + """ + + try self.gossipSimulationTest( + runs: 10, + setUpPeers: { () in + [ + Cluster.Gossip.parse(initialGossipState, owner: systemA.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialGossipState, owner: systemB.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialGossipState, owner: systemC.cluster.node, nodes: self.nodes), + ] + }, + updateLogic: { logics in + let logicA: MembershipGossipLogic = self.logic("A") + + // We simulate that `A` noticed it's the leader and moved `B` and `C` .up + logicA.localGossipUpdate(gossip: Cluster.Gossip.parse( + """ + A.up B.up C.up + A: A@5 B@3 C@3 + B: A@3 B@3 C@3 + C: A@3 B@3 C@3 + """, + owner: systemA.cluster.node, nodes: nodes + )) + }, + stopRunWhen: { (logics, results) in + logics.allSatisfy { $0.latestGossip.converged() } + }, + assert: { results in + results.roundCounts.max()!.shouldBeLessThanOrEqual(3) // usually 2 but 3 is tolerable; may be 1 if we're very lucky with ordering + results.messageCounts.max()!.shouldBeLessThanOrEqual(9) // usually 6, but 9 is tolerable + } + ) + } + + func test_shouldEventuallySuspendGossiping() throws { + let systemA = self.setUpNode("A") { settings in + settings.cluster.enabled = true + } + let systemB = self.setUpNode("B") + let systemC = self.setUpNode("C") + + let initialGossipState = + """ + A.up B.joining C.joining + A: A@3 B@3 C@3 + B: A@3 B@3 C@3 + C: A@3 B@3 C@3 + """ + + try self.gossipSimulationTest( + runs: 10, + setUpPeers: { () in + [ + Cluster.Gossip.parse(initialGossipState, owner: systemA.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialGossipState, owner: systemB.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialGossipState, owner: systemC.cluster.node, nodes: self.nodes), + ] + }, + updateLogic: { logics in + let logicA: MembershipGossipLogic = self.logic("A") + + // We simulate that `A` noticed it's the leader and moved `B` and `C` .up + logicA.localGossipUpdate(gossip: Cluster.Gossip.parse( + """ + A.up B.up C.up + A: A@5 B@3 C@3 + B: A@3 B@3 C@3 + C: A@3 B@3 C@3 + """, + owner: systemA.cluster.node, nodes: nodes + )) + }, + stopRunWhen: { logics, results in + logics.allSatisfy { logic in + logic.selectPeers(peers: self.peers(of: logic)) == [] // no more peers to talk to + } + }, + assert: { results in + results.roundCounts.max()!.shouldBeLessThanOrEqual(3) // usually 2 but 3 is tolerable; may be 1 if we're very lucky with ordering + results.messageCounts.max()!.shouldBeLessThanOrEqual(9) // usually 6, but up to 9 is tolerable + } + ) + } + + // ==== ---------------------------------------------------------------------------------------------------------------- + // MARK: Simulation test infra + + func gossipSimulationTest( + runs: Int, + setUpPeers: () -> [Cluster.Gossip], + updateLogic: ([MembershipGossipLogic]) -> (), + stopRunWhen: ([MembershipGossipLogic], GossipSimulationResults) -> Bool, + assert: (GossipSimulationResults) -> () + ) throws { + var roundCounts: [Int] = [] + var messageCounts: [Int] = [] + + var results = GossipSimulationResults( + runs: 0, + roundCounts: roundCounts, + messageCounts: messageCounts + ) + + let initialGossips = setUpPeers() + self.mockPeers = try! self.systems.map { system -> ActorRef.Message> in + let ref: ActorRef.Message> = + try system.spawn("peer", .receiveMessage { _ in .same }) + return self.systems.first!._resolveKnownRemote(ref, onRemoteSystem: system) + }.map { $0.asAddressable() } + + var log = self.systems.first!.log + log[metadataKey: "actor/path"] = "/user/peer" // mock actor path for log capture + + for runNr in 1...runs { + // initialize with user provided gossips + self.logics = initialGossips.map { initialGossip in + let system = self.system(initialGossip.owner.node.systemName) + let probe = self.testKit(system).spawnTestProbe(expecting: Cluster.Gossip.self) + let logic = self.makeLogic(system, probe) + logic.localGossipUpdate(gossip: initialGossip) + return logic + } + + func allConverged(gossips: [Cluster.Gossip]) -> Bool { + var allSatisfied = true // on purpose not via .allSatisfy() since we want to print status of each logic + for g in gossips.sorted(by: { $0.owner.node.systemName < $1.owner.node.systemName }) { + let converged = g.converged() + let convergenceStatus = converged ? "(locally assumed) converged" : "not converged" + + log.notice("\(g.owner.node.systemName): \(convergenceStatus)", metadata: [ + "gossip": Logger.MetadataValue.pretty(g) + ]) + + allSatisfied = allSatisfied && converged + } + return allSatisfied + } + + func simulateGossipRound() { + messageCounts.append(0) // make a counter for this run + + // we shuffle the gossips to simulate the slight timing differences -- not always will the "first" node be the first where the timers trigger + // and definitely not always will it always _remain_ the first to be gossiping; there may be others still gossiping around spreading their "not super complete" + // information. + let participatingGossips = self.logics.shuffled() + for logic in participatingGossips { + let selectedPeers: [AddressableActorRef] = logic.selectPeers(peers: self.peers(of: logic)) + log.notice("[\(logic.nodeName)] selected peers: \(selectedPeers.map({$0.address.node!.node.systemName}))") + + for targetPeer in selectedPeers { + messageCounts[messageCounts.endIndex - 1] += 1 + + let targetGossip = logic.makePayload(target: targetPeer) + if let gossip = targetGossip { + log.notice(" \(logic.nodeName) -> \(targetPeer.address.node!.node.systemName)", metadata: [ + "gossip": Logger.MetadataValue.pretty(gossip) + ]) + + let targetLogic = selectLogic(targetPeer) + let maybeAck = targetLogic.receiveGossip(gossip: gossip, from: self.peer(logic)) + log.notice("updated [\(targetPeer.address.node!.node.systemName)]", metadata: [ + "gossip": Logger.MetadataValue.pretty(targetLogic.latestGossip) + ]) + + if let ack = maybeAck { + log.notice(" \(logic.nodeName) <- \(targetPeer.address.node!.node.systemName) (ack)", metadata: [ + "ack": Logger.MetadataValue.pretty(ack) + ]) + logic.receiveAcknowledgement(from: self.peer(targetLogic), acknowledgement: ack, confirmsDeliveryOf: gossip) + } + + } else { + () // skipping target... + } + } + } + } + + updateLogic(logics) + + var rounds = 0 + log.notice("~~~~~~~~~~~~ new gossip instance ~~~~~~~~~~~~") + while !stopRunWhen(self.logics, results) { + rounds += 1 + log.notice("Next gossip round (\(rounds))...") + simulateGossipRound() + + if rounds > 20 { + fatalError("Too many gossip rounds detected! This is definitely wrong.") + } + + results = .init( + runs: runs, + roundCounts: roundCounts, + messageCounts: messageCounts + ) + } + + roundCounts += [rounds] + } + + pinfo("Finished [\(runs)] simulationsRounds: \(roundCounts)") + pinfo(" Rounds: \(roundCounts) (\(Double(roundCounts.reduce(0, +)) / Double(runs)) avg)") + pinfo(" Messages: \(messageCounts) (\(Double(messageCounts.reduce(0, +)) / Double(runs)) avg)") + + assert(results) + } + + struct GossipSimulationResults { + let runs: Int + var roundCounts: [Int] + var messageCounts: [Int] + } + + // ==== ---------------------------------------------------------------------------------------------------------------- + // MARK: Support functions + + func peers(of logic: MembershipGossipLogic) -> [AddressableActorRef] { + Array(self.mockPeers.filter { $0.address.node! != logic.localNode }) + } + + func selectLogic(_ peer: AddressableActorRef) -> MembershipGossipLogic { + guard let uniqueNode = peer.address.node else { + fatalError("MUST have node, was: \(peer.address)") + } + + return (self.logics.first { $0.localNode == uniqueNode })! + } + + func peer(_ logic: MembershipGossipLogic) -> AddressableActorRef { + let nodeName = logic.localNode.node.systemName + if let peer = (self.mockPeers.first { $0.address.node?.node.systemName == nodeName }) { + return peer + } else { + fatalError("No addressable peer for logic: \(logic), peers: \(self.mockPeers)") + } + } +} \ No newline at end of file diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift index 0c66c9d9e..52a6a03bb 100644 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift @@ -59,6 +59,10 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { [self.logicA, self.logicB, self.logicC] } + var gossips: [Cluster.Gossip] { + self.logics.map { $0.latestGossip } + } + override func setUp() { super.setUp() self.systemA = setUpNode("A") { settings in @@ -111,7 +115,7 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { """, owner: systemA.cluster.node, nodes: nodes ) - logicA.localGossipUpdate(payload: gossip) + logicA.localGossipUpdate(gossip: gossip) @@ -119,122 +123,26 @@ final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { round1.shouldEqual([self.b]) } - func test_stopCondition_converged() throws { - let gossip = Cluster.Gossip.parse( - """ - A.up B.joining C.up - A: A@5 B@5 C@6 - B: A@5 B@5 C@6 - C: A@5 B@5 C@6 - """, - owner: systemA.cluster.node, nodes: nodes - ) - logicA.localGossipUpdate(payload: gossip) - - let round1 = logicA.selectPeers(peers: self.peers(of: logicA)) - round1.shouldBeEmpty() - } - - func test_avgRounds_untilConvergence() throws { - let simulations = 10 - var roundCounts: [Int] = [] - var messageCounts: [Int] = Array(repeating: 0, count: simulations) - for simulationNr in 1...simulations { - self.initializeLogics() - - var gossipA = Cluster.Gossip.parse( - """ - A.up B.up C.up - A: A@5 B@3 C@3 - B: A@3 B@3 C@3 - C: A@3 B@3 C@3 - """, - owner: systemA.cluster.node, nodes: nodes - ) - logicA.localGossipUpdate(payload: gossipA) - - var gossipB = Cluster.Gossip.parse( - """ - A.up B.joining C.joining - A: A@3 B@3 C@3 - B: A@3 B@3 C@3 - C: A@3 B@3 C@3 - """, - owner: systemB.cluster.node, nodes: nodes - ) - logicB.localGossipUpdate(payload: gossipB) - - var gossipC = Cluster.Gossip.parse( - """ - A.up B.joining C.joining - A: A@3 B@3 C@3 - B: A@3 B@3 C@3 - C: A@3 B@3 C@3 - """, - owner: systemC.cluster.node, nodes: nodes - ) - logicC.localGossipUpdate(payload: gossipC) - - func allConverged(gossips: [Cluster.Gossip]) -> Bool { - var allSatisfied = true // on purpose not via .allSatisfy since we want to print status of each logic - for g in gossips.sorted(by: { $0.owner.node.systemName < $1.owner.node.systemName }) { - let converged = g.converged() - let convergenceStatus = converged ? "(locally assumed) converged" : "not converged" - pinfo("\(g.owner.node.systemName): \(convergenceStatus)") - allSatisfied = allSatisfied && converged - } - return allSatisfied - } - - func simulateGossipRound() { - // we shuffle the gossips to simulate the slight timing differences -- not always will the "first" node be the first where the timers trigger - // and definitely not always will it always _remain_ the first to be gossiping; there may be others still gossiping around spreading their "not super complete" - // information. - let participatingGossips = logics.shuffled() - for logic in participatingGossips { - let selectedPeers: [AddressableActorRef] = logic.selectPeers(peers: self.peers(of: logic)) - pinfo("[\(logic.nodeName)] selected peers: \(selectedPeers.map({$0.address.node!.node.systemName}))") - - for targetPeer in selectedPeers { - messageCounts[simulationNr - 1] += 1 - - let targetGossip = logic.makePayload(target: targetPeer) - if let gossip = targetGossip { - // pinfo(" \(logic.nodeName) -> \(targetPeer.address.node!.node.systemName): \(pretty: gossip)") - pinfo(" \(logic.nodeName) -> \(targetPeer.address.node!.node.systemName)") - - let targetLogic = selectLogic(targetPeer) - targetLogic.receiveGossip(origin: self.origin(logic), payload: gossip) - - pinfo("updated [\(targetPeer.address.node!.node.systemName)] gossip: \(targetLogic.latestGossip)") - switch targetPeer.address.node!.node.systemName { - case "A": gossipA = targetLogic.latestGossip - case "B": gossipB = targetLogic.latestGossip - case "C": gossipC = targetLogic.latestGossip - default: fatalError("No gossip storage space for \(targetPeer)") - } - } else { - () // skipping target... - } - } - } - } - - var rounds = 0 - pnote("~~~~~~~~~~~~ new gossip instance ~~~~~~~~~~~~") - while !allConverged(gossips: [gossipA, gossipB, gossipC]) { - rounds += 1 - pnote("Next gossip round (\(rounds))...") - simulateGossipRound() - } - - pnote("All peers converged after: [\(rounds) rounds]") - roundCounts += [rounds] - } - pnote("Finished [\(simulations)] simulationsRounds: \(roundCounts)") - pnote(" Rounds: \(roundCounts) (\(Double(roundCounts.reduce(0, +)) / Double(simulations)) avg)") - pnote(" Messages: \(messageCounts) (\(Double(messageCounts.reduce(0, +)) / Double(simulations)) avg)") - } +// func test_eventuallyStopGossiping() throws { +// let gossip = Cluster.Gossip.parse( +// """ +// A.up B.joining C.up +// A: A@5 B@5 C@6 +// B: A@5 B@5 C@6 +// C: A@5 B@5 C@6 +// """, +// owner: systemA.cluster.node, nodes: nodes +// ) +// logicA.localGossipUpdate(gossip: gossip) +// +// var rounds = 0 +// while logicA.sele { +// pprint("...") +// rounds += 1 +// } +// +// rounds.shouldBeLessThanOrEqual(10) +// } func test_logic_peersChanged() throws { let all = [a, b, c] From be6ff59ef02fa052b755bf8c20d6e9d40d40f312 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Tue, 23 Jun 2020 19:15:14 +0900 Subject: [PATCH 05/15] +cluster add simulated gossip test with many nodes --- ...MembershipGossipLogicSimulationTests.swift | 128 ++++++++++- .../Cluster/MembershipGossipLogicTests.swift | 203 ------------------ 2 files changed, 124 insertions(+), 207 deletions(-) delete mode 100644 Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift index fb837e54e..842ed1d7b 100644 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift @@ -121,6 +121,120 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas ) } + func test_avgRounds_manyNodes() throws { + let systemA = self.setUpNode("A") { settings in + settings.cluster.enabled = true + } + let systemB = self.setUpNode("B") + let systemC = self.setUpNode("C") + let systemD = self.setUpNode("D") + let systemE = self.setUpNode("E") + let systemF = self.setUpNode("F") + let systemG = self.setUpNode("G") + let systemH = self.setUpNode("H") + let systemI = self.setUpNode("I") + let systemJ = self.setUpNode("J") + + let allSystems = [ + systemA, systemB, systemC, systemD, systemE, + systemF, systemG, systemH, systemI, systemJ, + + ] + + let initialFewGossip = + """ + A.up B.joining C.joining D.joining E.joining F.joining G.joining H.joining I.joining J.joining + A: A@9 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@5 + B: A@5 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@5 + C: A@3 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@5 + D: A@2 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@1 + E: A@2 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@1 + F: A@2 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@1 + G: A@2 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@1 + H: A@2 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@1 + I: A@2 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@1 + J: A@2 B@3 C@3 D@5 E@5 F@5 G@5 H@5 I@5 J@1 + """ + let initialNewGossip = + """ + D.joining E.joining F.joining G.joining H.joining I.joining J.joining + D: D@5 E@5 F@5 G@5 H@5 I@5 J@5 + E: D@5 E@5 F@5 G@5 H@5 I@5 J@5 + F: D@5 E@5 F@5 G@5 H@5 I@5 J@5 + G: D@5 E@5 F@5 G@5 H@5 I@5 J@5 + H: D@5 E@5 F@5 G@5 H@5 I@5 J@5 + I: D@5 E@5 F@5 G@5 H@5 I@5 J@5 + J: D@5 E@5 F@5 G@5 H@5 I@5 J@5 + """ + + try self.gossipSimulationTest( + runs: 1, + setUpPeers: { () in + [ + Cluster.Gossip.parse(initialFewGossip, owner: systemA.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialFewGossip, owner: systemB.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialFewGossip, owner: systemC.cluster.node, nodes: self.nodes), + + Cluster.Gossip.parse(initialNewGossip, owner: systemD.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialNewGossip, owner: systemE.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialNewGossip, owner: systemF.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialNewGossip, owner: systemG.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialNewGossip, owner: systemH.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialNewGossip, owner: systemI.cluster.node, nodes: self.nodes), + Cluster.Gossip.parse(initialNewGossip, owner: systemJ.cluster.node, nodes: self.nodes), + ] + }, + updateLogic: { logics in + let logicA: MembershipGossipLogic = self.logic("A") + let logicD: MembershipGossipLogic = self.logic("D") + + logicA.localGossipUpdate(gossip: Cluster.Gossip.parse( + """ + A.up B.up C.up D.up E.up F.up G.up H.up I.up J.up + A: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + B: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + C: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + D: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + E: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + F: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + G: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + H: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + I: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + J: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 + """, + owner: systemA.cluster.node, nodes: nodes + )) + + // they're trying to join + logicD.localGossipUpdate(gossip: Cluster.Gossip.parse( + """ + A.up B.up C.up D.joining E.joining F.joining G.joining H.joining I.joining J.joining + A: A@11 B@16 C@16 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + B: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + C: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + D: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + E: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + F: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + G: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + H: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + I: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + J: A@12 B@11 C@11 D@9 E@13 F@13 G@13 H@13 I@13 J@13 + """, + owner: systemD.cluster.node, nodes: nodes + )) + }, + stopRunWhen: { (logics, results) in + // keep gossiping until all members become .up and converged + logics.allSatisfy { $0.latestGossip.converged() } && + logics.allSatisfy { $0.latestGossip.membership.count(withStatus: .up) == allSystems.count } + }, + assert: { results in + results.roundCounts.max()?.shouldBeLessThanOrEqual(3) + results.messageCounts.max()?.shouldBeLessThanOrEqual(10) + } + ) + } + func test_shouldEventuallySuspendGossiping() throws { let systemA = self.setUpNode("A") { settings in settings.cluster.enabled = true @@ -165,8 +279,8 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas } }, assert: { results in - results.roundCounts.max()!.shouldBeLessThanOrEqual(3) // usually 2 but 3 is tolerable; may be 1 if we're very lucky with ordering - results.messageCounts.max()!.shouldBeLessThanOrEqual(9) // usually 6, but up to 9 is tolerable + results.roundCounts.max()!.shouldBeLessThanOrEqual(4) + results.messageCounts.max()!.shouldBeLessThanOrEqual(12) } ) } @@ -200,7 +314,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas var log = self.systems.first!.log log[metadataKey: "actor/path"] = "/user/peer" // mock actor path for log capture - for runNr in 1...runs { + for _ in 1...runs { // initialize with user provided gossips self.logics = initialGossips.map { initialGossip in let system = self.system(initialGossip.owner.node.systemName) @@ -288,7 +402,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas roundCounts += [rounds] } - pinfo("Finished [\(runs)] simulationsRounds: \(roundCounts)") + pinfo("Finished [\(runs)] simulation runs") pinfo(" Rounds: \(roundCounts) (\(Double(roundCounts.reduce(0, +)) / Double(runs)) avg)") pinfo(" Messages: \(messageCounts) (\(Double(messageCounts.reduce(0, +)) / Double(runs)) avg)") @@ -324,4 +438,10 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas fatalError("No addressable peer for logic: \(logic), peers: \(self.mockPeers)") } } +} + +fileprivate extension MembershipGossipLogic { + var nodeName: String { + self.localNode.node.systemName + } } \ No newline at end of file diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift deleted file mode 100644 index 52a6a03bb..000000000 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicTests.swift +++ /dev/null @@ -1,203 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Distributed Actors open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the Swift Distributed Actors project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.md for the list of Swift Distributed Actors project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -@testable import DistributedActors -import DistributedActorsTestKit -import NIO -import XCTest - -final class MembershipGossipLogicTests: ClusteredActorSystemsXCTestCase { - override func configureActorSystem(settings: inout ActorSystemSettings) { - settings.cluster.enabled = false // not actually clustering, just need a few nodes - } - - override func configureLogCapture(settings: inout LogCapture.Settings) { - settings.excludeActorPaths = [ - "/system/replicator/gossip", - "/system/replicator", - "/system/swim", - "/system/clusterEvents", - "/system/cluster", - "/system/leadership", - ] - } - - lazy var systemA: ActorSystem! = nil - lazy var systemB: ActorSystem! = nil - lazy var systemC: ActorSystem! = nil - var systems: [ActorSystem] = [] - var nodes: [UniqueNode] = [] - var allPeers: [AddressableActorRef] = [] - - lazy var a: AddressableActorRef = self.allPeers.first { $0.address.node?.node.systemName == "A" }! - lazy var b: AddressableActorRef = self.allPeers.first { $0.address.node?.node.systemName == "B" }! - lazy var c: AddressableActorRef = self.allPeers.first { $0.address.node?.node.systemName == "C" }! - - lazy var testKit: ActorTestKit! = nil - - lazy var pA: ActorTestProbe! = nil - lazy var logicA: MembershipGossipLogic! = nil - - lazy var pB: ActorTestProbe! = nil - lazy var logicB: MembershipGossipLogic! = nil - - lazy var pC: ActorTestProbe! = nil - lazy var logicC: MembershipGossipLogic! = nil - - var logics: [MembershipGossipLogic] { - [self.logicA, self.logicB, self.logicC] - } - - var gossips: [Cluster.Gossip] { - self.logics.map { $0.latestGossip } - } - - override func setUp() { - super.setUp() - self.systemA = setUpNode("A") { settings in - settings.cluster.enabled = true // to allow remote resolves, though we never send messages there - } - self.systemB = setUpNode("B") - self.systemC = setUpNode("C") - - self.systems = [systemA, systemB, systemC] - self.nodes = systems.map { $0.cluster.node } - self.allPeers = try! systems.map { system -> ActorRef.Message> in - let ref: ActorRef.Message> = try system.spawn("peer", .receiveMessage { _ in .same }) - return self.systemA._resolveKnownRemote(ref, onRemoteSystem: system) - }.map { $0.asAddressable() } - - self.testKit = self.testKit(self.systemA) - - self.pA = testKit.spawnTestProbe(expecting: Cluster.Gossip.self) - self.pB = testKit.spawnTestProbe(expecting: Cluster.Gossip.self) - self.pC = testKit.spawnTestProbe(expecting: Cluster.Gossip.self) - initializeLogics() - } - - private func initializeLogics() { - self.logicA = makeLogic(self.systemA, self.pA) - self.logicB = makeLogic(self.systemB, self.pB) - self.logicC = makeLogic(self.systemC, self.pC) - } - - private func makeLogic(_ system: ActorSystem, _ probe: ActorTestProbe) -> MembershipGossipLogic { - MembershipGossipLogic( - GossipLogicContext( - ownerContext: self.testKit(system).makeFakeContext(), - gossipIdentifier: StringGossipIdentifier("membership") - ), - notifyOnGossipRef: probe.ref - ) - } - - // ==== ------------------------------------------------------------------------------------------------------------ - // MARK: Tests - - func test_pickMostBehindNode() throws { - let gossip = Cluster.Gossip.parse( - """ - A.up B.joining C.up - A: A@5 B@5 C@6 - B: A@5 B@1 C@1 - C: A@5 B@5 C@6 - """, - owner: systemA.cluster.node, nodes: nodes - ) - logicA.localGossipUpdate(gossip: gossip) - - - - let round1 = logicA.selectPeers(peers: self.peers(of: logicA)) - round1.shouldEqual([self.b]) - } - -// func test_eventuallyStopGossiping() throws { -// let gossip = Cluster.Gossip.parse( -// """ -// A.up B.joining C.up -// A: A@5 B@5 C@6 -// B: A@5 B@5 C@6 -// C: A@5 B@5 C@6 -// """, -// owner: systemA.cluster.node, nodes: nodes -// ) -// logicA.localGossipUpdate(gossip: gossip) -// -// var rounds = 0 -// while logicA.sele { -// pprint("...") -// rounds += 1 -// } -// -// rounds.shouldBeLessThanOrEqual(10) -// } - - func test_logic_peersChanged() throws { - let all = [a, b, c] - let known: [AddressableActorRef] = [a] - let less: [AddressableActorRef] = [] - let more: [AddressableActorRef] = [a, b] - - let res1 = MembershipGossipLogic.peersChanged(known: known, current: less) - res1!.removed.shouldEqual([a]) - res1!.added.shouldEqual([]) - - let res2 = MembershipGossipLogic.peersChanged(known: known, current: more) - res2!.removed.shouldEqual([]) - res2!.added.shouldEqual([b]) - - let res3 = MembershipGossipLogic.peersChanged(known: [], current: all) - res3!.removed.shouldEqual([]) - res3!.added.shouldEqual([a, b, c]) - } - - // ==== ---------------------------------------------------------------------------------------------------------------- - // MARK: Support functions - - func peers(of logic: MembershipGossipLogic) -> [AddressableActorRef] { - Array(self.allPeers.filter { $0.address.node! != logic.localNode }) - } - - func selectLogic(_ peer: AddressableActorRef) -> MembershipGossipLogic { - guard let node = peer.address.node else { - fatalError("MUST have node, was: \(peer.address)") - } - - switch node.node.systemName { - case "A": return self.logicA - case "B": return self.logicB - case "C": return self.logicC - default: fatalError("No logic for peer: \(peer)") - } - } - - func origin(_ logic: MembershipGossipLogic) -> AddressableActorRef { - if ObjectIdentifier(logic) == ObjectIdentifier(logicA) { - return self.a - } else if ObjectIdentifier(logic) == ObjectIdentifier(logicB) { - return self.b - } else if ObjectIdentifier(logic) == ObjectIdentifier(logicC) { - return self.c - } else { - fatalError("No addressable peer for logic: \(logic)") - } - } -} - -extension MembershipGossipLogic { - var nodeName: String { - self.localNode.node.systemName - } -} \ No newline at end of file From 6a7565bb389cc1064d1f2705e7f7894621fa0ef5 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Tue, 23 Jun 2020 20:02:20 +0900 Subject: [PATCH 06/15] =gossip,fix gossiper MUST watch any peer it communicates with in order to remove it upon termination --- .../Cluster+MembershipGossipLogic.swift | 9 +- .../Gossip/Gossip+Shell.swift | 5 ++ .../Gossip/GossipShellTests.swift | 88 +++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 Tests/DistributedActorsTests/Gossip/GossipShellTests.swift diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift index ca414b2f9..79fa8aea4 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift @@ -30,6 +30,9 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { /// We store and use a shuffled yet stable order for gossiping peers. /// See `updateActivePeers` for details. private var peers: [AddressableActorRef] = [] + /// Constantly mutated by `nextPeerToGossipWith` in an effort to keep order in which we gossip with nodes evenly distributed. + /// This follows our logic in SWIM, and has the benefit that we never get too chatty with one specific node (as in the worst case it may be unreachable or down already). + private var _peerToGossipWithIndex: Int = 0 /// During 1:1 gossip interactions, update this table, which means "we definitely know the specific node has seen our version VV at ..." /// @@ -108,6 +111,9 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { } /// True if the peers is "behind" in terms of information it has "seen" (as determined by comparing our and its seen tables). + // TODO: Implement stricter-round robin, the same way as our SWIM impl does, see `nextMemberToPing` + // This hardens the implementation against gossiping with the same node multiple times in a row. + // Note that we do NOT need to worry about filtering out dead peers as this is automatically handled by the gossip shell. private func shouldGossipWith(_ peer: AddressableActorRef) -> Bool { guard let remoteNode = peer.address.node else { // targets should always be remote peers; one not having a node should not happen, let's ignore it as a gossip target @@ -119,9 +125,6 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { return true } -// pprint("\(self.localNode.node.systemName) = self.latestGossip.version = \(pretty: self.latestGossip)") -// pprint("\(self.localNode.node.systemName) = lastSeenGossipFromPeer[\(peer.address.node!.node.systemName)] = \(pretty: lastSeenGossipFromPeer)") - // TODO: can be replaced by a digest comparison return self.latestGossip.seen != lastSeenGossipFromPeer.seen } diff --git a/Sources/DistributedActors/Gossip/Gossip+Shell.swift b/Sources/DistributedActors/Gossip/Gossip+Shell.swift index 7515d372c..1b4cc2587 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Shell.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Shell.swift @@ -263,6 +263,11 @@ extension GossipShell { let resolved: AddressableActorRef = resolvePeerOn(member) if let peer = resolved.ref as? PeerRef { + // We MUST always watch all peers we gossip with, as if they (or their nodes) were to terminate + // they MUST be removed from the peer list we offer to gossip logics. Otherwise a naive gossip logic + // may continue trying to gossip with that peer. + context.watch(peer) + if self.peers.insert(peer).inserted { context.log.debug("Automatically discovered peer", metadata: [ "gossip/peer": "\(peer)", diff --git a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift b/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift new file mode 100644 index 000000000..7f1f2be3f --- /dev/null +++ b/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift @@ -0,0 +1,88 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Distributed Actors open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the Swift Distributed Actors project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.md for the list of Swift Distributed Actors project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@testable import DistributedActors +import DistributedActorsTestKit +import Foundation +import NIOSSL +import XCTest + +final class GossipShellTests: ActorSystemXCTestCase { + + func test_down_beGossipedToOtherNodes() throws { + let p = self.testKit.spawnTestProbe(expecting: [AddressableActorRef].self) + + let control = try Gossiper.start( + self.system, + name: "gossiper", + settings: .init(gossipInterval: .seconds(1)), + makeLogic: { _ in InspectOfferedPeersTestGossipLogic(offeredPeersProbe: p.ref) } + ) + + let peerBehavior: Behavior.Message> = .receiveMessage { msg in + if "\(msg)".contains("stop") { return .stop } else { return .same } + } + let first = try self.system.spawn("first", peerBehavior) + let second = try self.system.spawn("second", peerBehavior) + + control.introduce(peer: first) + control.introduce(peer: second) + control.update(StringGossipIdentifier("hi"), payload: .init("hello")) + + try Set(p.expectMessage()).shouldEqual(Set([first.asAddressable(), second.asAddressable()])) + + first.tell(.removePayload(identifier: StringGossipIdentifier("stop"))) + try Set(p.expectMessage()).shouldEqual(Set([second.asAddressable()])) + + first.tell(.removePayload(identifier: StringGossipIdentifier("stop"))) + try p.expectNoMessage(for: .milliseconds(300)) + } + struct InspectOfferedPeersTestGossipLogic: GossipLogic { + struct Envelope: GossipEnvelopeProtocol { + let metadata: String + let payload: String + + init(_ info: String) { + self.metadata = info + self.payload = info + } + } + typealias Acknowledgement = String + + let offeredPeersProbe: ActorRef<[AddressableActorRef]> + init(offeredPeersProbe: ActorRef<[AddressableActorRef]>) { + self.offeredPeersProbe = offeredPeersProbe + } + + func selectPeers(peers: [AddressableActorRef]) -> [AddressableActorRef] { + self.offeredPeersProbe.tell(peers) + return [] + } + + func makePayload(target: AddressableActorRef) -> Envelope? { + nil + } + + func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) { + } + + func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? { + nil + } + + func localGossipUpdate(gossip: Envelope) { + } + } + +} \ No newline at end of file From 4f41ebe1071a44eeb2e61c991212ef4b4d3c03dd Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Tue, 23 Jun 2020 22:40:59 +0900 Subject: [PATCH 07/15] =gossip,membership an incomin ACK must be treated as any other gossip, since it may cause events, or it may cause convergence to trigger which means if we're a leader we must perform our duties --- .../Cluster/Cluster+Gossip.swift | 10 ----- .../Cluster/Cluster+Membership.swift | 4 +- .../Cluster+MembershipGossipLogic.swift | 44 ++++++------------- .../Cluster/ClusterShell.swift | 3 +- .../Gossip/Gossip+Shell.swift | 7 +-- .../ClusteredActorSystemsXCTestCase.swift | 7 +-- ...MembershipGossipLogicSimulationTests.swift | 44 +++++++++---------- .../Gossip/GossipShellTests.swift | 12 +++-- 8 files changed, 51 insertions(+), 80 deletions(-) diff --git a/Sources/DistributedActors/Cluster/Cluster+Gossip.swift b/Sources/DistributedActors/Cluster/Cluster+Gossip.swift index bc9b1b356..dac5d1346 100644 --- a/Sources/DistributedActors/Cluster/Cluster+Gossip.swift +++ b/Sources/DistributedActors/Cluster/Cluster+Gossip.swift @@ -26,16 +26,6 @@ extension Cluster { /// It may have in the mean time of course observed a new version already. // TODO: There is tons of compression opportunity about not having to send full tables around in general, but for now we will just send them around // FIXME: ensure that we never have a seen entry for a non-member - // bad: "actor/message": Gossip( - // owner: sact://first:2342486320@127.0.0.1:9001, - // seen: Cluster.Gossip.SeenTable( - // [sact://second:4264003847@127.0.0.1:9002: [uniqueNode:sact://second@127.0.0.1:9002: 2], - // sact://first:2342486320@127.0.0.1:9001: [uniqueNode:sact://first@127.0.0.1:9001: 4, uniqueNode:sact://second@127.0.0.1:9002: 2]] - // ), - // membership: Membership(count: 2, leader: Member(sact://first@127.0.0.1:9001, status: joining, reachability: reachable), - // members: [ - // Member(sact://first:2342486320@127.0.0.1:9001, status: joining, reachability: reachable), - // Member(sact://second-REPLACEMENT:871659343@127.0.0.1:9002, status: joining, reachability: reachable)])) var seen: Cluster.Gossip.SeenTable /// The version vector of this gossip and the `Membership` state owned by it. var version: VersionVector { diff --git a/Sources/DistributedActors/Cluster/Cluster+Membership.swift b/Sources/DistributedActors/Cluster/Cluster+Membership.swift index 3652a4dd7..52a006c1d 100644 --- a/Sources/DistributedActors/Cluster/Cluster+Membership.swift +++ b/Sources/DistributedActors/Cluster/Cluster+Membership.swift @@ -238,9 +238,9 @@ extension Cluster.Membership: Hashable { extension Cluster.Membership: CustomStringConvertible, CustomDebugStringConvertible, CustomPrettyStringConvertible { /// Pretty multi-line output of a membership, useful for manual inspection public var prettyDescription: String { - var res = "LEADER: \(self.leader, orElse: ".none")" + var res = "leader: \(self.leader, orElse: ".none")" for member in self._members.values.sorted(by: { $0.node.node.port < $1.node.node.port }) { - res += "\n \(reflecting: member.node) STATUS: [\(member.status.rawValue, leftPadTo: Cluster.MemberStatus.maxStrLen)]" + res += "\n \(reflecting: member.node) status [\(member.status.rawValue, leftPadTo: Cluster.MemberStatus.maxStrLen)]" } return res } diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift index 79fa8aea4..cb986d1ae 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift @@ -102,30 +102,21 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { self.latestGossip } - func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Cluster.Gossip) { - // 1) store the direct gossip we got from this peer; we can use this to know if there's no need to gossip to that peer by inspecting seen table equality - self.lastGossipFrom[peer] = acknowledgement - - // 2) use this to move forward the gossip as well - self.mergeInbound(gossip: acknowledgement) - } - /// True if the peers is "behind" in terms of information it has "seen" (as determined by comparing our and its seen tables). // TODO: Implement stricter-round robin, the same way as our SWIM impl does, see `nextMemberToPing` // This hardens the implementation against gossiping with the same node multiple times in a row. // Note that we do NOT need to worry about filtering out dead peers as this is automatically handled by the gossip shell. private func shouldGossipWith(_ peer: AddressableActorRef) -> Bool { - guard let remoteNode = peer.address.node else { - // targets should always be remote peers; one not having a node should not happen, let's ignore it as a gossip target - return false - } +// guard let remoteNode = peer.address.node else { +// // targets should always be remote peers; one not having a node should not happen, let's ignore it as a gossip target +// return false +// } guard let lastSeenGossipFromPeer = self.lastGossipFrom[peer] else { // it's a peer we have not gotten any gossip from yet return true } - // TODO: can be replaced by a digest comparison return self.latestGossip.seen != lastSeenGossipFromPeer.seen } @@ -177,27 +168,20 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { return self.latestGossip } - func localGossipUpdate(gossip: Cluster.Gossip) { - self.mergeInbound(gossip: gossip) - } + func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Cluster.Gossip) { + // 1) store the direct gossip we got from this peer; we can use this to know if there's no need to gossip to that peer by inspecting seen table equality + self.lastGossipFrom[peer] = acknowledgement - // ==== ------------------------------------------------------------------------------------------------------------ - // MARK: Side-channel + // 2) move forward the gossip we store + self.mergeInbound(gossip: acknowledgement) - enum SideChannelMessage { - case localUpdate(Envelope) + // 3) notify listeners + self.notifyOnGossipRef.tell(self.latestGossip) } - func receiveSideChannelMessage(message: Any) throws { - guard let sideChannelMessage = message as? SideChannelMessage else { - self.context.system.deadLetters.tell(DeadLetter(message, recipient: self.context.gossiperAddress)) - return - } - - switch sideChannelMessage { - case .localUpdate(let gossip): - self.mergeInbound(gossip: gossip) - } + func localGossipUpdate(gossip: Cluster.Gossip) { + self.mergeInbound(gossip: gossip) + self.context.log.info("MERGED local gossip update: \(pretty: self.latestGossip)") } // ==== ------------------------------------------------------------------------------------------------------------ diff --git a/Sources/DistributedActors/Cluster/ClusterShell.swift b/Sources/DistributedActors/Cluster/ClusterShell.swift index cba2915d5..47d9f6be1 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell.swift @@ -551,7 +551,8 @@ extension ClusterShell { var state = state let changeDirective = state.applyClusterEvent(event) - state = self.interpretLeaderActions(context.system, state, state.collectLeaderActions()) + let actions: [ClusterShellState.LeaderAction] = state.collectLeaderActions() + state = self.interpretLeaderActions(context.system, state, actions) if case .membershipChange(let change) = event { self.tryIntroduceGossipPeer(context, state, change: change) diff --git a/Sources/DistributedActors/Gossip/Gossip+Shell.swift b/Sources/DistributedActors/Gossip/Gossip+Shell.swift index 1b4cc2587..b02420285 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Shell.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Shell.swift @@ -113,14 +113,11 @@ internal final class GossipShell, identifier: GossipIdentifier) -> AnyGossipLogic { diff --git a/Sources/DistributedActorsTestKit/Cluster/ClusteredActorSystemsXCTestCase.swift b/Sources/DistributedActorsTestKit/Cluster/ClusteredActorSystemsXCTestCase.swift index 7505a5ac0..728256c21 100644 --- a/Sources/DistributedActorsTestKit/Cluster/ClusteredActorSystemsXCTestCase.swift +++ b/Sources/DistributedActorsTestKit/Cluster/ClusteredActorSystemsXCTestCase.swift @@ -123,7 +123,8 @@ open class ClusteredActorSystemsXCTestCase: XCTestCase { public func joinNodes( node: ActorSystem, with other: ActorSystem, - ensureWithin: TimeAmount? = nil, ensureMembers maybeExpectedStatus: Cluster.MemberStatus? = nil + ensureWithin: TimeAmount? = nil, ensureMembers maybeExpectedStatus: Cluster.MemberStatus? = nil, + file: StaticString = #file, line: UInt = #line ) throws { node.cluster.join(node: other.cluster.node.node) @@ -132,9 +133,9 @@ open class ClusteredActorSystemsXCTestCase: XCTestCase { if let expectedStatus = maybeExpectedStatus { if let specificTimeout = ensureWithin { - try self.ensureNodes(expectedStatus, on: node, within: specificTimeout, nodes: other.cluster.node) + try self.ensureNodes(expectedStatus, on: node, within: specificTimeout, nodes: other.cluster.node, file: file, line: line) } else { - try self.ensureNodes(expectedStatus, on: node, nodes: other.cluster.node) + try self.ensureNodes(expectedStatus, on: node, nodes: other.cluster.node, file: file, line: line) } } } diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift index 842ed1d7b..4a21b1c4f 100644 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift @@ -14,9 +14,9 @@ @testable import DistributedActors import DistributedActorsTestKit +import Logging import NIO import XCTest -import Logging final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCase { override func configureActorSystem(settings: inout ActorSystemSettings) { @@ -25,13 +25,14 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas override func configureLogCapture(settings: inout LogCapture.Settings) { settings.filterActorPaths = [ - "/user/peer" + "/user/peer", ] } var systems: [ActorSystem] { self._nodes } + func system(_ id: String) -> ActorSystem { self.systems.first(where: { $0.name == id })! } @@ -97,7 +98,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas Cluster.Gossip.parse(initialGossipState, owner: systemC.cluster.node, nodes: self.nodes), ] }, - updateLogic: { logics in + updateLogic: { _ in let logicA: MembershipGossipLogic = self.logic("A") // We simulate that `A` noticed it's the leader and moved `B` and `C` .up @@ -111,7 +112,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas owner: systemA.cluster.node, nodes: nodes )) }, - stopRunWhen: { (logics, results) in + stopRunWhen: { (logics, _) in logics.allSatisfy { $0.latestGossip.converged() } }, assert: { results in @@ -138,7 +139,6 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas let allSystems = [ systemA, systemB, systemC, systemD, systemE, systemF, systemG, systemH, systemI, systemJ, - ] let initialFewGossip = @@ -184,7 +184,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas Cluster.Gossip.parse(initialNewGossip, owner: systemJ.cluster.node, nodes: self.nodes), ] }, - updateLogic: { logics in + updateLogic: { _ in let logicA: MembershipGossipLogic = self.logic("A") let logicD: MembershipGossipLogic = self.logic("D") @@ -223,10 +223,10 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas owner: systemD.cluster.node, nodes: nodes )) }, - stopRunWhen: { (logics, results) in + stopRunWhen: { (logics, _) in // keep gossiping until all members become .up and converged logics.allSatisfy { $0.latestGossip.converged() } && - logics.allSatisfy { $0.latestGossip.membership.count(withStatus: .up) == allSystems.count } + logics.allSatisfy { $0.latestGossip.membership.count(withStatus: .up) == allSystems.count } }, assert: { results in results.roundCounts.max()?.shouldBeLessThanOrEqual(3) @@ -259,7 +259,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas Cluster.Gossip.parse(initialGossipState, owner: systemC.cluster.node, nodes: self.nodes), ] }, - updateLogic: { logics in + updateLogic: { _ in let logicA: MembershipGossipLogic = self.logic("A") // We simulate that `A` noticed it's the leader and moved `B` and `C` .up @@ -273,7 +273,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas owner: systemA.cluster.node, nodes: nodes )) }, - stopRunWhen: { logics, results in + stopRunWhen: { logics, _ in logics.allSatisfy { logic in logic.selectPeers(peers: self.peers(of: logic)) == [] // no more peers to talk to } @@ -291,9 +291,9 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas func gossipSimulationTest( runs: Int, setUpPeers: () -> [Cluster.Gossip], - updateLogic: ([MembershipGossipLogic]) -> (), + updateLogic: ([MembershipGossipLogic]) -> Void, stopRunWhen: ([MembershipGossipLogic], GossipSimulationResults) -> Bool, - assert: (GossipSimulationResults) -> () + assert: (GossipSimulationResults) -> Void ) throws { var roundCounts: [Int] = [] var messageCounts: [Int] = [] @@ -314,12 +314,12 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas var log = self.systems.first!.log log[metadataKey: "actor/path"] = "/user/peer" // mock actor path for log capture - for _ in 1...runs { + for _ in 1 ... runs { // initialize with user provided gossips self.logics = initialGossips.map { initialGossip in let system = self.system(initialGossip.owner.node.systemName) let probe = self.testKit(system).spawnTestProbe(expecting: Cluster.Gossip.self) - let logic = self.makeLogic(system, probe) + let logic = self.makeLogic(system, probe) logic.localGossipUpdate(gossip: initialGossip) return logic } @@ -331,7 +331,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas let convergenceStatus = converged ? "(locally assumed) converged" : "not converged" log.notice("\(g.owner.node.systemName): \(convergenceStatus)", metadata: [ - "gossip": Logger.MetadataValue.pretty(g) + "gossip": Logger.MetadataValue.pretty(g), ]) allSatisfied = allSatisfied && converged @@ -348,7 +348,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas let participatingGossips = self.logics.shuffled() for logic in participatingGossips { let selectedPeers: [AddressableActorRef] = logic.selectPeers(peers: self.peers(of: logic)) - log.notice("[\(logic.nodeName)] selected peers: \(selectedPeers.map({$0.address.node!.node.systemName}))") + log.notice("[\(logic.nodeName)] selected peers: \(selectedPeers.map { $0.address.node!.node.systemName })") for targetPeer in selectedPeers { messageCounts[messageCounts.endIndex - 1] += 1 @@ -356,18 +356,18 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas let targetGossip = logic.makePayload(target: targetPeer) if let gossip = targetGossip { log.notice(" \(logic.nodeName) -> \(targetPeer.address.node!.node.systemName)", metadata: [ - "gossip": Logger.MetadataValue.pretty(gossip) + "gossip": Logger.MetadataValue.pretty(gossip), ]) - let targetLogic = selectLogic(targetPeer) + let targetLogic = self.selectLogic(targetPeer) let maybeAck = targetLogic.receiveGossip(gossip: gossip, from: self.peer(logic)) log.notice("updated [\(targetPeer.address.node!.node.systemName)]", metadata: [ - "gossip": Logger.MetadataValue.pretty(targetLogic.latestGossip) + "gossip": Logger.MetadataValue.pretty(targetLogic.latestGossip), ]) if let ack = maybeAck { log.notice(" \(logic.nodeName) <- \(targetPeer.address.node!.node.systemName) (ack)", metadata: [ - "ack": Logger.MetadataValue.pretty(ack) + "ack": Logger.MetadataValue.pretty(ack), ]) logic.receiveAcknowledgement(from: self.peer(targetLogic), acknowledgement: ack, confirmsDeliveryOf: gossip) } @@ -440,8 +440,8 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas } } -fileprivate extension MembershipGossipLogic { +private extension MembershipGossipLogic { var nodeName: String { self.localNode.node.systemName } -} \ No newline at end of file +} diff --git a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift b/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift index 7f1f2be3f..ddb54d04a 100644 --- a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift +++ b/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift @@ -19,7 +19,6 @@ import NIOSSL import XCTest final class GossipShellTests: ActorSystemXCTestCase { - func test_down_beGossipedToOtherNodes() throws { let p = self.testKit.spawnTestProbe(expecting: [AddressableActorRef].self) @@ -48,6 +47,7 @@ final class GossipShellTests: ActorSystemXCTestCase { first.tell(.removePayload(identifier: StringGossipIdentifier("stop"))) try p.expectNoMessage(for: .milliseconds(300)) } + struct InspectOfferedPeersTestGossipLogic: GossipLogic { struct Envelope: GossipEnvelopeProtocol { let metadata: String @@ -58,6 +58,7 @@ final class GossipShellTests: ActorSystemXCTestCase { self.payload = info } } + typealias Acknowledgement = String let offeredPeersProbe: ActorRef<[AddressableActorRef]> @@ -74,15 +75,12 @@ final class GossipShellTests: ActorSystemXCTestCase { nil } - func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) { - } + func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) {} func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? { nil } - func localGossipUpdate(gossip: Envelope) { - } + func localGossipUpdate(gossip: Envelope) {} } - -} \ No newline at end of file +} From 897da3bc3590247079702585d30e5b1e885e686a Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Tue, 23 Jun 2020 23:01:08 +0900 Subject: [PATCH 08/15] =cluster fix existing handshake detection on incoming; Only ASSOCIATED for eagerly rejecting --- .../Cluster+MembershipGossipLogic.swift | 8 +++---- .../Cluster/ClusterShellState.swift | 23 ++++++++++++++----- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift index cb986d1ae..bae253998 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift @@ -107,10 +107,10 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { // This hardens the implementation against gossiping with the same node multiple times in a row. // Note that we do NOT need to worry about filtering out dead peers as this is automatically handled by the gossip shell. private func shouldGossipWith(_ peer: AddressableActorRef) -> Bool { -// guard let remoteNode = peer.address.node else { -// // targets should always be remote peers; one not having a node should not happen, let's ignore it as a gossip target -// return false -// } + guard peer.address.node != nil else { + // targets should always be remote peers; one not having a node should not happen, let's ignore it as a gossip target + return false + } guard let lastSeenGossipFromPeer = self.lastGossipFrom[peer] else { // it's a peer we have not gotten any gossip from yet diff --git a/Sources/DistributedActors/Cluster/ClusterShellState.swift b/Sources/DistributedActors/Cluster/ClusterShellState.swift index 94e5ebbaa..591c8c387 100644 --- a/Sources/DistributedActors/Cluster/ClusterShellState.swift +++ b/Sources/DistributedActors/Cluster/ClusterShellState.swift @@ -251,12 +251,23 @@ extension ClusterShellState { return .negotiateIncoming(fsm) } - guard existingAssociation == nil else { - let error = HandshakeStateMachine.HandshakeConnectionError( - node: offer.originNode.node, - message: "Terminating this connection, the node [\(offer.originNode)] is already associated. Possibly a delayed handshake retry message was delivered?" - ) - return .abortIncomingHandshake(error) + if let assoc = existingAssociation { + switch assoc.state { + case .associating: + () // continue, we'll perform the tie-breaker logic below + case .associated: + let error = HandshakeStateMachine.HandshakeConnectionError( + node: offer.originNode.node, + message: "Terminating this connection, the node [\(offer.originNode)] is already associated. Possibly a delayed handshake retry message was delivered?" + ) + return .abortIncomingHandshake(error) + case .tombstone: + let error = HandshakeStateMachine.HandshakeConnectionError( + node: offer.originNode.node, + message: "Terminating this connection, the node [\(offer.originNode)] is already tombstone-ed. Possibly a delayed handshake retry message was delivered?" + ) + return .abortIncomingHandshake(error) + } } guard let inProgress = self._handshakes[offer.originNode.node] else { From 079232500b3b08f40c8bc52e1cf2a7d61a5a1b48 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Tue, 23 Jun 2020 23:27:58 +0900 Subject: [PATCH 09/15] cleanup --- .../Cluster/Cluster+MembershipGossipLogic.swift | 1 - .../DistributedActorsTestKit/ShouldMatchers.swift | 12 ++++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift index bae253998..a330cd1d7 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift @@ -181,7 +181,6 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { func localGossipUpdate(gossip: Cluster.Gossip) { self.mergeInbound(gossip: gossip) - self.context.log.info("MERGED local gossip update: \(pretty: self.latestGossip)") } // ==== ------------------------------------------------------------------------------------------------------------ diff --git a/Sources/DistributedActorsTestKit/ShouldMatchers.swift b/Sources/DistributedActorsTestKit/ShouldMatchers.swift index bc070a714..83f5b5bcf 100644 --- a/Sources/DistributedActorsTestKit/ShouldMatchers.swift +++ b/Sources/DistributedActorsTestKit/ShouldMatchers.swift @@ -43,13 +43,17 @@ public struct TestMatchers { public extension TestMatchers where T: Equatable { func toEqual(_ expected: T) { - let error = self.callSite.notEqualError(got: self.it, expected: expected) - XCTAssertEqual(self.it, expected, "\(error)", file: self.callSite.file, line: self.callSite.line) + if self.it != expected { + let error = self.callSite.notEqualError(got: self.it, expected: expected) + XCTFail("\(error)", file: self.callSite.file, line: self.callSite.line) + } } func toNotEqual(_ unexpectedEqual: T) { - let error = self.callSite.equalError(got: self.it, unexpectedEqual: unexpectedEqual) - XCTAssertNotEqual(self.it, unexpectedEqual, "\(error)", file: self.callSite.file, line: self.callSite.line) + if self.it == unexpectedEqual { + let error = self.callSite.equalError(got: self.it, unexpectedEqual: unexpectedEqual) + XCTFail("\(error)", file: self.callSite.file, line: self.callSite.line) + } } func toBe(_ expected: Other.Type) { From 19739d721a13ef5eb7e2bc10dc535384821291d8 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Wed, 24 Jun 2020 13:28:14 +0900 Subject: [PATCH 10/15] review followup --- .../DistributedActors/CRDT/CRDT+Gossip.swift | 12 ++-- .../CRDT/CRDT+ReplicatorShell.swift | 4 +- .../Cluster/ClusterShell+LeaderActions.swift | 2 +- .../Cluster/ClusterShell+Logging.swift | 2 +- .../Cluster/ClusterShell.swift | 20 +++--- .../Cluster/ClusterShellState.swift | 16 ++--- ...ster+MembershipGossip+Serialization.swift} | 10 +-- .../Cluster+MembershipGossip.swift} | 24 +++---- .../Cluster+MembershipGossipLogic.swift | 28 ++++---- .../Gossip/Gossip+Logic.swift | 55 ++++++++-------- .../Gossip/Gossip+Serialization.swift | 2 +- .../Gossip/Gossip+Settings.swift | 2 +- .../Gossip/Gossip+Shell.swift | 42 ++++++------ .../Serialization/Serialization.swift | 4 +- .../Cluster/ClusterLeaderActionsTests.swift | 2 +- .../DowningClusteredTests.swift | 2 +- .../Cluster/GossipSeenTableTests.swift | 34 +++++----- ...MembershipGossipLogicSimulationTests.swift | 64 +++++++++---------- .../Cluster/MembershipGossipTests.swift | 64 +++++++++---------- .../Membership+SerializationTests.swift | 4 +- .../TestExtensions+MembershipDSL.swift | 14 ++-- .../Cluster/TestExtensions.swift | 2 +- .../Gossip/GossipShellTests.swift | 12 ++-- 23 files changed, 209 insertions(+), 212 deletions(-) rename Sources/DistributedActors/Cluster/{Cluster+Gossip+Serialization.swift => MembershipGossip/Cluster+MembershipGossip+Serialization.swift} (93%) rename Sources/DistributedActors/Cluster/{Cluster+Gossip.swift => MembershipGossip/Cluster+MembershipGossip.swift} (94%) rename Sources/DistributedActors/Cluster/{ => MembershipGossip}/Cluster+MembershipGossipLogic.swift (89%) diff --git a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift index 538608523..dffe5f3c8 100644 --- a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift +++ b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift @@ -20,7 +20,7 @@ extension CRDT { /// It collaborates with the Direct Replicator in order to avoid needlessly sending values to nodes which already know /// about them (e.g. through direct replication). final class GossipReplicatorLogic: GossipLogic { - typealias Envelope = CRDT.Gossip + typealias Gossip = CRDT.Gossip typealias Acknowledgement = CRDT.GossipAck let identity: CRDT.Identity @@ -80,7 +80,7 @@ extension CRDT { self.latest } - func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: CRDT.Gossip) { + func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming envelope: CRDT.Gossip) { guard (self.latest.map { $0.payload.equalState(to: envelope.payload) } ?? false) else { // received an ack for something, however it's not the "latest" anymore, so we need to gossip to target anyway return @@ -94,7 +94,7 @@ extension CRDT { // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> CRDT.GossipAck? { + func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> CRDT.GossipAck? { // merge the datatype locally, and update our information about the origin's knowledge about this datatype // (does it already know about our data/all-deltas-we-are-aware-of or not) self.mergeInbound(from: peer, gossip) @@ -106,7 +106,7 @@ extension CRDT { return CRDT.GossipAck() } - func localGossipUpdate(gossip: CRDT.Gossip) { + func receiveLocalGossipUpdate(_ gossip: CRDT.Gossip) { self.mergeInbound(from: nil, gossip) // during the next gossip round we'll gossip the latest most-up-to date version now; // no need to schedule that, we'll be called when it's time. @@ -120,8 +120,8 @@ extension CRDT { // and gossip replicator, so that's a downside that we'll eventually want to address. enum SideChannelMessage { - case localUpdate(Envelope) - case ack(origin: AddressableActorRef, payload: Envelope) + case localUpdate(Gossip) + case ack(origin: AddressableActorRef, payload: Gossip) } func receiveSideChannelMessage(message: Any) throws { diff --git a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift index de353572c..f90d96b55 100644 --- a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift +++ b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift @@ -33,7 +33,7 @@ extension CRDT.Replicator { typealias RemoteDeleteResult = CRDT.Replicator.RemoteCommand.DeleteResult private let directReplicator: CRDT.Replicator.Instance - private var gossipReplication: GossipControl! + private var gossipReplication: GossiperControl! // TODO: better name; this is the control from Gossip -> Local struct LocalControl { @@ -74,8 +74,6 @@ extension CRDT.Replicator { self.gossipReplication = try Gossiper.start( context, name: "gossip", - of: CRDT.Gossip.self, - ofAcknowledgement: CRDT.GossipAck.self, settings: Gossiper.Settings( gossipInterval: self.settings.gossipInterval, gossipIntervalRandomFactor: self.settings.gossipIntervalRandomFactor, diff --git a/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift b/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift index 60d26a966..ec9e0666a 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift @@ -145,7 +145,7 @@ extension ClusterShell { return } state._latestGossip.incrementOwnerVersion() - state.gossipControl.update(payload: state._latestGossip) + state.gossiperControl.update(payload: state._latestGossip) self.terminateAssociation(system, state: &state, memberToRemove.node) diff --git a/Sources/DistributedActors/Cluster/ClusterShell+Logging.swift b/Sources/DistributedActors/Cluster/ClusterShell+Logging.swift index 8cba021f7..c7f074a1a 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell+Logging.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell+Logging.swift @@ -85,7 +85,7 @@ extension ClusterShell { case send(to: Node) case receive(from: Node) case receiveUnique(from: UniqueNode) - case gossip(Cluster.Gossip) + case gossip(Cluster.MembershipGossip) var description: String { switch self { diff --git a/Sources/DistributedActors/Cluster/ClusterShell.swift b/Sources/DistributedActors/Cluster/ClusterShell.swift index 47d9f6be1..f4fe58b87 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell.swift @@ -314,7 +314,7 @@ internal class ClusterShell { /// Gossiping is handled by /system/cluster/gossip, however acting on it still is our task, /// thus the gossiper forwards gossip whenever interesting things happen ("more up to date gossip") /// to the shell, using this message, so we may act on it -- e.g. perform leader actions or change membership that we store. - case gossipFromGossiper(Cluster.Gossip) + case gossipFromGossiper(Cluster.MembershipGossip) } // this is basically our API internally for this system @@ -423,7 +423,7 @@ extension ClusterShell { context.log.info("Bound to \(chan.localAddress.map { $0.description } ?? "")") // TODO: Membership.Gossip? - let gossipControl: GossipControl = try Gossiper.start( + let gossiperControl: GossiperControl = try Gossiper.start( context, name: "\(ActorPath._clusterGossip.name)", props: ._wellKnown, @@ -431,14 +431,14 @@ extension ClusterShell { gossipInterval: clusterSettings.membershipGossipInterval, gossipIntervalRandomFactor: clusterSettings.membershipGossipIntervalRandomFactor, peerDiscovery: .onClusterMember(atLeast: .joining, resolve: { member in - let resolveContext = ResolveContext.Message>(address: ._clusterGossip(on: member.node), system: context.system) + let resolveContext = ResolveContext.Message>(address: ._clusterGossip(on: member.node), system: context.system) return context.system._resolve(context: resolveContext).asAddressable() }) ), makeLogic: { MembershipGossipLogic( $0, - notifyOnGossipRef: context.messageAdapter(from: Cluster.Gossip.self) { + notifyOnGossipRef: context.messageAdapter(from: Cluster.MembershipGossip.self) { Optional.some(Message.gossipFromGossiper($0)) } ) @@ -449,17 +449,17 @@ extension ClusterShell { settings: clusterSettings, channel: chan, events: self.clusterEvents, - gossipControl: gossipControl, + gossiperControl: gossiperControl, log: context.log ) // loop through "self" cluster shell, which in result causes notifying all subscribers about cluster membership change - var firstGossip = Cluster.Gossip(ownerNode: state.localNode) + var firstGossip = Cluster.MembershipGossip(ownerNode: state.localNode) _ = firstGossip.membership.join(state.localNode) // change will be put into effect by receiving the "self gossip" firstGossip.incrementOwnerVersion() context.system.cluster.updateMembershipSnapshot(state.membership) - gossipControl.update(payload: firstGossip) // ???? + gossiperControl.update(payload: firstGossip) // ???? context.myself.tell(.gossipFromGossiper(firstGossip)) // TODO: are we ok if we received another gossip first, not our own initial? should be just fine IMHO @@ -572,7 +572,7 @@ extension ClusterShell { func receiveMembershipGossip( _ context: ActorContext, _ state: ClusterShellState, - gossip: Cluster.Gossip + gossip: Cluster.MembershipGossip ) -> Behavior { tracelog(context, .gossip(gossip), message: gossip) var state = state @@ -637,12 +637,12 @@ extension ClusterShell { // TODO: make it cleaner? though we decided to go with manual peer management as the ClusterShell owns it, hm // TODO: consider receptionist instead of this; we're "early" but receptionist could already be spreading its info to this node, since we associated. - let gossipPeer: GossipShell.Ref = context.system._resolve( + let gossipPeer: GossipShell.Ref = context.system._resolve( context: .init(address: ._clusterGossip(on: change.member.node), system: context.system) ) // FIXME: make sure that if the peer terminated, we don't add it again in here, receptionist would be better then to power this... // today it can happen that a node goes down but we dont know yet so we add it again :O - state.gossipControl.introduce(peer: gossipPeer) + state.gossiperControl.introduce(peer: gossipPeer) } } diff --git a/Sources/DistributedActors/Cluster/ClusterShellState.swift b/Sources/DistributedActors/Cluster/ClusterShellState.swift index 591c8c387..033bd0a08 100644 --- a/Sources/DistributedActors/Cluster/ClusterShellState.swift +++ b/Sources/DistributedActors/Cluster/ClusterShellState.swift @@ -60,14 +60,14 @@ internal struct ClusterShellState: ReadOnlyClusterState { internal var _handshakes: [Node: HandshakeStateMachine.State] = [:] - let gossipControl: GossipControl + let gossiperControl: GossiperControl /// Updating the `latestGossip` causes the gossiper to be informed about it, such that the next time it does a gossip round /// it uses the latest gossip available. - var _latestGossip: Cluster.Gossip + var _latestGossip: Cluster.MembershipGossip /// Any change to the gossip data, is propagated to the gossiper immediately. - var latestGossip: Cluster.Gossip { + var latestGossip: Cluster.MembershipGossip { get { self._latestGossip } @@ -75,7 +75,7 @@ internal struct ClusterShellState: ReadOnlyClusterState { if self._latestGossip.membership == newValue.membership { self._latestGossip = newValue } else { - let next: Cluster.Gossip + let next: Cluster.MembershipGossip if self._latestGossip.version == newValue.version { next = newValue.incrementingOwnerVersion() } else { @@ -84,7 +84,7 @@ internal struct ClusterShellState: ReadOnlyClusterState { self._latestGossip = next } - self.gossipControl.update(payload: self._latestGossip) + self.gossiperControl.update(payload: self._latestGossip) } } @@ -101,7 +101,7 @@ internal struct ClusterShellState: ReadOnlyClusterState { settings: ClusterSettings, channel: Channel, events: EventStream, - gossipControl: GossipControl, + gossiperControl: GossiperControl, log: Logger ) { self.log = log @@ -110,10 +110,10 @@ internal struct ClusterShellState: ReadOnlyClusterState { self.eventLoopGroup = settings.eventLoopGroup ?? settings.makeDefaultEventLoopGroup() self.localNode = settings.uniqueBindNode - self._latestGossip = Cluster.Gossip(ownerNode: settings.uniqueBindNode) + self._latestGossip = Cluster.MembershipGossip(ownerNode: settings.uniqueBindNode) self.events = events - self.gossipControl = gossipControl + self.gossiperControl = gossiperControl self.channel = channel } diff --git a/Sources/DistributedActors/Cluster/Cluster+Gossip+Serialization.swift b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip+Serialization.swift similarity index 93% rename from Sources/DistributedActors/Cluster/Cluster+Gossip+Serialization.swift rename to Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip+Serialization.swift index a28636aa4..4a133046f 100644 --- a/Sources/DistributedActors/Cluster/Cluster+Gossip+Serialization.swift +++ b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip+Serialization.swift @@ -12,7 +12,7 @@ // //===----------------------------------------------------------------------===// -extension Cluster.Gossip: ProtobufRepresentable { +extension Cluster.MembershipGossip: ProtobufRepresentable { typealias ProtobufRepresentation = ProtoClusterMembershipGossip public func toProto(context: Serialization.Context) throws -> ProtobufRepresentation { @@ -36,13 +36,13 @@ extension Cluster.Gossip: ProtobufRepresentable { public init(fromProto proto: ProtobufRepresentation, context: Serialization.Context) throws { guard proto.ownerUniqueNodeID != 0 else { - throw SerializationError.missingField("ownerUniqueNodeID", type: "\(reflecting: Cluster.Gossip.self)") + throw SerializationError.missingField("ownerUniqueNodeID", type: "\(reflecting: Cluster.MembershipGossip.self)") } guard proto.hasMembership else { - throw SerializationError.missingField("membership", type: "\(reflecting: Cluster.Gossip.self)") + throw SerializationError.missingField("membership", type: "\(reflecting: Cluster.MembershipGossip.self)") } guard proto.hasSeenTable else { - throw SerializationError.missingField("seenTable", type: "\(reflecting: Cluster.Gossip.self)") + throw SerializationError.missingField("seenTable", type: "\(reflecting: Cluster.MembershipGossip.self)") } let membership = try Cluster.Membership(fromProto: proto.membership, context: context) @@ -52,7 +52,7 @@ extension Cluster.Gossip: ProtobufRepresentable { throw SerializationError.unableToDeserialize(hint: "Missing member for ownerUniqueNodeID, members: \(membership)") } - var gossip = Cluster.Gossip(ownerNode: ownerNode) + var gossip = Cluster.MembershipGossip(ownerNode: ownerNode) gossip.membership = membership gossip.seen.underlying.reserveCapacity(proto.seenTable.rows.count) for row in proto.seenTable.rows { diff --git a/Sources/DistributedActors/Cluster/Cluster+Gossip.swift b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift similarity index 94% rename from Sources/DistributedActors/Cluster/Cluster+Gossip.swift rename to Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift index dac5d1346..f2da92902 100644 --- a/Sources/DistributedActors/Cluster/Cluster+Gossip.swift +++ b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift @@ -19,14 +19,14 @@ extension Cluster { /// Gossip payload about members in the cluster. /// /// Used to guarantee phrases like "all nodes have seen a node A in status S", upon which the Leader may act. - struct Gossip: ActorMessage, Equatable { + struct MembershipGossip: ActorMessage, Equatable { let owner: UniqueNode /// A table maintaining our perception of other nodes views on the version of membership. /// Each row in the table represents what versionVector we know the given node has observed recently. /// It may have in the mean time of course observed a new version already. // TODO: There is tons of compression opportunity about not having to send full tables around in general, but for now we will just send them around // FIXME: ensure that we never have a seen entry for a non-member - var seen: Cluster.Gossip.SeenTable + var seen: Cluster.MembershipGossip.SeenTable /// The version vector of this gossip and the `Membership` state owned by it. var version: VersionVector { self.seen.underlying[self.owner]! // !-safe, since we _always_ know our own world view @@ -39,7 +39,7 @@ extension Cluster { init(ownerNode: UniqueNode) { self.owner = ownerNode // self.seen = Cluster.Gossip.SeenTable(myselfNode: ownerNode, version: VersionVector((.uniqueNode(ownerNode), 1))) - self.seen = Cluster.Gossip.SeenTable(myselfNode: ownerNode, version: VersionVector()) + self.seen = Cluster.MembershipGossip.SeenTable(myselfNode: ownerNode, version: VersionVector()) // The actual payload self.membership = .empty // MUST be empty, as on the first "self gossip, we cause all ClusterEvents @@ -58,7 +58,7 @@ extension Cluster { /// Merge an incoming gossip _into_ the current gossip. /// Ownership of this gossip is retained, versions are bumped, and membership is merged. - mutating func mergeForward(incoming: Gossip) -> MergeDirective { + mutating func mergeForward(incoming: MembershipGossip) -> MergeDirective { var incoming = incoming // 1) decide the relationship between this gossip and the incoming one @@ -134,7 +134,7 @@ extension Cluster { /// Only `.up` and `.leaving` members are considered, since joining members are "too early" /// to matter in decisions, and down members shall never participate in decision making. func converged() -> Bool { - let members = self.membership.members(withStatus: [.joining, .up, .leaving]) // FIXME: we should not require joining nodes in convergence, can losen up a bit here I hope + let members = self.membership.members(withStatus: [.up, .leaving]) // FIXME: we should not require joining nodes in convergence, can losen up a bit here I hope let requiredVersion = self.version if members.isEmpty { @@ -158,13 +158,9 @@ extension Cluster { } } -// struct GossipAck: Codable { -// let owner: UniqueNode -// var seen: Cluster.Gossip.SeenTable -// } } -extension Cluster.Gossip: GossipEnvelopeProtocol { +extension Cluster.MembershipGossip: GossipEnvelopeProtocol { typealias Metadata = SeenTable typealias Payload = Self @@ -177,12 +173,12 @@ extension Cluster.Gossip: GossipEnvelopeProtocol { } } -extension Cluster.Gossip: CustomPrettyStringConvertible {} +extension Cluster.MembershipGossip: CustomPrettyStringConvertible {} // ==== ---------------------------------------------------------------------------------------------------------------- // MARK: Cluster.Gossip.SeenTable -extension Cluster.Gossip { +extension Cluster.MembershipGossip { /// A table containing information about which node has seen the gossip at which version. /// /// It is best visualized as a series of views (by "owners" of a row) onto the state of the cluster. @@ -295,9 +291,9 @@ extension Cluster.Gossip { } } -extension Cluster.Gossip.SeenTable: CustomStringConvertible, CustomPrettyStringConvertible { +extension Cluster.MembershipGossip.SeenTable: CustomStringConvertible, CustomPrettyStringConvertible { public var description: String { - "Cluster.Gossip.SeenTable(\(self.underlying))" + "Cluster.MembershipGossip.SeenTable(\(self.underlying))" } public func prettyDescription(depth: Int) -> String { diff --git a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossipLogic.swift similarity index 89% rename from Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift rename to Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossipLogic.swift index a330cd1d7..95260e8ec 100644 --- a/Sources/DistributedActors/Cluster/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossipLogic.swift @@ -17,15 +17,19 @@ import NIO // ==== ---------------------------------------------------------------------------------------------------------------- // MARK: Membership Gossip Logic +/// The logic of the membership gossip. +/// +/// Membership gossip is what is used to reach cluster "convergence" upon which a leader may perform leader actions. +/// See `Cluster.MembershipGossip.converged` for more details. final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { - typealias Envelope = Cluster.Gossip - typealias Acknowledgement = Cluster.Gossip // TODO: GossipAck instead, a more minimal one; just the peers status + typealias Gossip = Cluster.MembershipGossip + typealias Acknowledgement = Cluster.MembershipGossip private let context: Context internal lazy var localNode: UniqueNode = self.context.system.cluster.node - internal var latestGossip: Cluster.Gossip - private let notifyOnGossipRef: ActorRef + internal var latestGossip: Cluster.MembershipGossip + private let notifyOnGossipRef: ActorRef /// We store and use a shuffled yet stable order for gossiping peers. /// See `updateActivePeers` for details. @@ -38,9 +42,9 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { /// /// See `updateActivePeers` and `receiveGossip` for details. // TODO: This can be optimized and it's enough if we keep a digest of the gossips; this way ACKs can just send the digest as well saving space. - private var lastGossipFrom: [AddressableActorRef: Cluster.Gossip] = [:] + private var lastGossipFrom: [AddressableActorRef: Cluster.MembershipGossip] = [:] - init(_ context: Context, notifyOnGossipRef: ActorRef) { + init(_ context: Context, notifyOnGossipRef: ActorRef) { self.context = context self.notifyOnGossipRef = notifyOnGossipRef self.latestGossip = .init(ownerNode: context.system.cluster.node) @@ -96,7 +100,7 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { } } - func makePayload(target: AddressableActorRef) -> Cluster.Gossip? { + func makePayload(target: AddressableActorRef) -> Cluster.MembershipGossip? { // today we don't trim payloads at all // TODO: trim some information? self.latestGossip @@ -153,7 +157,7 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? { + func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? { // 1) mark that from that specific peer, we know it observed at least that version self.lastGossipFrom[peer] = gossip @@ -168,7 +172,7 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { return self.latestGossip } - func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Cluster.Gossip) { + func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming gossip: Cluster.MembershipGossip) { // 1) store the direct gossip we got from this peer; we can use this to know if there's no need to gossip to that peer by inspecting seen table equality self.lastGossipFrom[peer] = acknowledgement @@ -179,14 +183,14 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { self.notifyOnGossipRef.tell(self.latestGossip) } - func localGossipUpdate(gossip: Cluster.Gossip) { + func receiveLocalGossipUpdate(_ gossip: Cluster.MembershipGossip) { self.mergeInbound(gossip: gossip) } // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Utilities - private func mergeInbound(gossip: Cluster.Gossip) { + private func mergeInbound(gossip: Cluster.MembershipGossip) { _ = self.latestGossip.mergeForward(incoming: gossip) // effects are signalled via the ClusterShell, not here (it will also perform a merge) // TODO: a bit duplicated, could we maintain it here? } @@ -201,7 +205,7 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { let MembershipGossipIdentifier: StringGossipIdentifier = "membership" -extension GossipControl where GossipEnvelope == Cluster.Gossip { +extension GossiperControl where GossipEnvelope == Cluster.MembershipGossip { func update(payload: GossipEnvelope) { self.update(MembershipGossipIdentifier, payload: payload) } diff --git a/Sources/DistributedActors/Gossip/Gossip+Logic.swift b/Sources/DistributedActors/Gossip/Gossip+Logic.swift index a5858458f..2e7d64fb2 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Logic.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Logic.swift @@ -38,10 +38,10 @@ import Logging /// for a nice overview of the general concepts involved in gossip algorithms. /// - SeeAlso: `Cluster.Gossip` for the Actor System's own gossip mechanism for membership dissemination public protocol GossipLogic { - associatedtype Envelope: GossipEnvelopeProtocol + associatedtype Gossip: GossipEnvelopeProtocol associatedtype Acknowledgement: Codable - typealias Context = GossipLogicContext + typealias Context = GossipLogicContext // init(context: Context) // TODO: a form of context? @@ -56,7 +56,7 @@ public protocol GossipLogic { // TODO: make a directive here /// Allows for customizing the payload for specific targets - mutating func makePayload(target: AddressableActorRef) -> Envelope? + mutating func makePayload(target: AddressableActorRef) -> Gossip? /// Invoked when the specific gossiped payload is acknowledged by the target. /// @@ -64,26 +64,25 @@ public protocol GossipLogic { /// Eg. if gossip is sent to 2 peers, it is NOT deterministic which of the acks returns first (or at all!). /// /// - Parameters: + /// - acknowledgement: acknowledgement sent by the peer /// - peer: The target which has acknowledged the gossiped payload. /// It corresponds to the parameter that was passed to the `makePayload(target:)` which created this gossip payload. - /// - acknowledgement: acknowledgement sent by the peer - /// - envelope: - mutating func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) + /// - gossip: + mutating func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming gossip: Gossip) // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Receiving gossip - mutating func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? + mutating func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? - mutating func localGossipUpdate(gossip: Envelope) + mutating func receiveLocalGossipUpdate(_ gossip: Gossip) /// Extra side channel, allowing for arbitrary outside interactions with this gossip logic. - // TODO: We could consider making it typed perhaps... - mutating func receiveSideChannelMessage(message: Any) throws + mutating func receiveSideChannelMessage(_ message: Any) throws } extension GossipLogic { - public mutating func receiveSideChannelMessage(message: Any) throws { + public mutating func receiveSideChannelMessage(_ message: Any) throws { // ignore by default } } @@ -117,52 +116,52 @@ public struct GossipLogicContext: GossipLogic, CustomStringConvertible { +public struct AnyGossipLogic: GossipLogic, CustomStringConvertible { @usableFromInline let _selectPeers: ([AddressableActorRef]) -> [AddressableActorRef] @usableFromInline - let _makePayload: (AddressableActorRef) -> Envelope? + let _makePayload: (AddressableActorRef) -> Gossip? @usableFromInline - let _receiveGossip: (Envelope, AddressableActorRef) -> Acknowledgement? + let _receiveGossip: (Gossip, AddressableActorRef) -> Acknowledgement? @usableFromInline - let _receiveAcknowledgement: (AddressableActorRef, Acknowledgement, Envelope) -> Void + let _receiveAcknowledgement: (Acknowledgement, AddressableActorRef, Gossip) -> Void @usableFromInline - let _localGossipUpdate: (Envelope) -> Void + let _receiveLocalGossipUpdate: (Gossip) -> Void @usableFromInline let _receiveSideChannelMessage: (Any) throws -> Void public init(_ logic: Logic) - where Logic: GossipLogic, Logic.Envelope == Envelope, Logic.Acknowledgement == Acknowledgement { + where Logic: GossipLogic, Logic.Gossip == Gossip, Logic.Acknowledgement == Acknowledgement { var l = logic self._selectPeers = { l.selectPeers(peers: $0) } self._makePayload = { l.makePayload(target: $0) } - self._receiveGossip = { l.receiveGossip(gossip: $0, from: $1) } + self._receiveGossip = { l.receiveGossip($0, from: $1) } - self._receiveAcknowledgement = { l.receiveAcknowledgement(from: $0, acknowledgement: $1, confirmsDeliveryOf: $2) } - self._localGossipUpdate = { l.localGossipUpdate(gossip: $0) } + self._receiveAcknowledgement = { l.receiveAcknowledgement($0, from: $1, confirming: $2) } + self._receiveLocalGossipUpdate = { l.receiveLocalGossipUpdate($0) } - self._receiveSideChannelMessage = { try l.receiveSideChannelMessage(message: $0) } + self._receiveSideChannelMessage = { try l.receiveSideChannelMessage($0) } } public func selectPeers(peers: [AddressableActorRef]) -> [AddressableActorRef] { self._selectPeers(peers) } - public func makePayload(target: AddressableActorRef) -> Envelope? { + public func makePayload(target: AddressableActorRef) -> Gossip? { self._makePayload(target) } - public func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? { + public func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? { self._receiveGossip(gossip, peer) } - public func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) { - self._receiveAcknowledgement(peer, acknowledgement, envelope) + public func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming envelope: Gossip) { + self._receiveAcknowledgement(acknowledgement, peer, envelope) } - public func localGossipUpdate(gossip: Envelope) { - self._localGossipUpdate(gossip) + public func receiveLocalGossipUpdate(_ gossip: Gossip) { + self._receiveLocalGossipUpdate(gossip) } public func receiveSideChannelMessage(_ message: Any) throws { @@ -170,7 +169,7 @@ public struct AnyGossipLogic(...)" + "\(reflecting: Self.self)(...)" } } diff --git a/Sources/DistributedActors/Gossip/Gossip+Serialization.swift b/Sources/DistributedActors/Gossip/Gossip+Serialization.swift index 19feabff9..f4855e734 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Serialization.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Serialization.swift @@ -51,7 +51,7 @@ extension GossipShell.Message: Codable { // FIXME: sometimes we could encode raw and not via the Data -- think about it and fix it let payloadManifest = try container.decode(Serialization.Manifest.self, forKey: .gossip_payload_manifest) let payloadPayload = try container.decode(Data.self, forKey: .gossip_payload) - let payload = try context.serialization.deserialize(as: Envelope.self, from: .data(payloadPayload), using: payloadManifest) + let payload = try context.serialization.deserialize(as: Gossip.self, from: .data(payloadPayload), using: payloadManifest) let ackRefAddress = try container.decode(ActorAddress.self, forKey: .ackRef) let ackRef = context.resolveActorRef(Acknowledgement.self, identifiedBy: ackRefAddress) diff --git a/Sources/DistributedActors/Gossip/Gossip+Settings.swift b/Sources/DistributedActors/Gossip/Gossip+Settings.swift index 5c54939c6..8d38329a5 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Settings.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Settings.swift @@ -47,7 +47,7 @@ extension Gossiper { case manuallyIntroduced /// Automatically register this gossiper and subscribe for any others identifying under the same - /// `Receptionist.RegistrationKey.Message>(id)`. + /// `Receptionist.RegistrationKey.Message>(id)`. case fromReceptionistListing(id: String) /// Automatically discover and add cluster members to the gossip group when they become reachable in `atLeast` status. diff --git a/Sources/DistributedActors/Gossip/Gossip+Shell.swift b/Sources/DistributedActors/Gossip/Gossip+Shell.swift index b02420285..3d4c996e3 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Shell.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Shell.swift @@ -17,15 +17,15 @@ import Logging private let gossipTickKey: TimerKey = "gossip-tick" /// Convergent gossip is a gossip mechanism which aims to equalize some state across all peers participating. -internal final class GossipShell { +internal final class GossipShell { typealias Ref = ActorRef let settings: Gossiper.Settings - private let makeLogic: (ActorContext, GossipIdentifier) -> AnyGossipLogic + private let makeLogic: (ActorContext, GossipIdentifier) -> AnyGossipLogic /// Payloads to be gossiped on gossip rounds - private var gossipLogics: [AnyGossipIdentifier: AnyGossipLogic] + private var gossipLogics: [AnyGossipIdentifier: AnyGossipLogic] typealias PeerRef = ActorRef private var peers: Set @@ -33,7 +33,7 @@ internal final class GossipShell( settings: Gossiper.Settings, makeLogic: @escaping (Logic.Context) -> Logic - ) where Logic: GossipLogic, Logic.Envelope == Envelope, Logic.Acknowledgement == Acknowledgement { + ) where Logic: GossipLogic, Logic.Gossip == Gossip, Logic.Acknowledgement == Acknowledgement { self.settings = settings self.makeLogic = { shellContext, id in let logicContext = GossipLogicContext(ownerContext: shellContext, gossipIdentifier: id) @@ -90,7 +90,7 @@ internal final class GossipShell, identifier: GossipIdentifier, origin: ActorRef, - payload: Envelope, + payload: Gossip, ackRef: ActorRef ) { context.log.trace("Received gossip [\(identifier.gossipIdentifier)]", metadata: [ @@ -101,7 +101,7 @@ internal final class GossipShell, identifier: GossipIdentifier, - payload: Envelope + payload: Gossip ) { let logic = self.getEnsureLogic(context, identifier: identifier) @@ -117,11 +117,11 @@ internal final class GossipShell, identifier: GossipIdentifier) -> AnyGossipLogic { - let logic: AnyGossipLogic + private func getEnsureLogic(_ context: ActorContext, identifier: GossipIdentifier) -> AnyGossipLogic { + let logic: AnyGossipLogic if let existing = self.gossipLogics[identifier.asAnyGossipIdentifier] { logic = existing } else { @@ -164,7 +164,7 @@ internal final class GossipShell, identifier: AnyGossipIdentifier, - _ payload: Envelope, + _ payload: Gossip, to target: PeerRef, onGossipAck: @escaping (Acknowledgement) -> Void ) { @@ -355,10 +355,10 @@ extension GossipShell { extension GossipShell { enum Message { // gossip - case gossip(identity: GossipIdentifier, origin: ActorRef, Envelope, ackRef: ActorRef) + case gossip(identity: GossipIdentifier, origin: ActorRef, Gossip, ackRef: ActorRef) // local messages - case updatePayload(identifier: GossipIdentifier, Envelope) + case updatePayload(identifier: GossipIdentifier, Gossip) case removePayload(identifier: GossipIdentifier) case introducePeer(PeerRef) @@ -382,8 +382,8 @@ public enum Gossiper { props: Props = .init(), settings: Settings = .init(), makeLogic: @escaping (Logic.Context) -> Logic - ) throws -> GossipControl - where Logic: GossipLogic, Logic.Envelope == Envelope, Logic.Acknowledgement == Acknowledgement { + ) throws -> GossiperControl + where Logic: GossipLogic, Logic.Gossip == Envelope, Logic.Acknowledgement == Acknowledgement { let ref = try context.spawn( naming, of: GossipShell.Message.self, @@ -391,14 +391,14 @@ public enum Gossiper { file: #file, line: #line, GossipShell(settings: settings, makeLogic: makeLogic).behavior ) - return GossipControl(ref) + return GossiperControl(ref) } } // ==== ---------------------------------------------------------------------------------------------------------------- -// MARK: GossipControl +// MARK: GossiperControl -internal struct GossipControl { +internal struct GossiperControl { private let ref: GossipShell.Ref init(_ ref: GossipShell.Ref) { diff --git a/Sources/DistributedActors/Serialization/Serialization.swift b/Sources/DistributedActors/Serialization/Serialization.swift index dc61ba47a..08f6ccc47 100644 --- a/Sources/DistributedActors/Serialization/Serialization.swift +++ b/Sources/DistributedActors/Serialization/Serialization.swift @@ -115,8 +115,8 @@ public class Serialization { // cluster settings.register(ClusterShell.Message.self) settings.register(Cluster.Event.self) - settings.register(Cluster.Gossip.self) - settings.register(GossipShell.Message.self) + settings.register(Cluster.MembershipGossip.self) + settings.register(GossipShell.Message.self) settings.register(StringGossipIdentifier.self) // receptionist needs some special casing diff --git a/Tests/DistributedActorsTests/Cluster/ClusterLeaderActionsTests.swift b/Tests/DistributedActorsTests/Cluster/ClusterLeaderActionsTests.swift index 1e5e5bf31..78ca910cc 100644 --- a/Tests/DistributedActorsTests/Cluster/ClusterLeaderActionsTests.swift +++ b/Tests/DistributedActorsTests/Cluster/ClusterLeaderActionsTests.swift @@ -149,7 +149,7 @@ final class ClusterLeaderActionsTests: XCTestCase { } @discardableResult - private func gossip(from: ClusterShellState, to: inout ClusterShellState) -> Cluster.Gossip.MergeDirective { + private func gossip(from: ClusterShellState, to: inout ClusterShellState) -> Cluster.MembershipGossip.MergeDirective { to.latestGossip.mergeForward(incoming: from.latestGossip) } diff --git a/Tests/DistributedActorsTests/Cluster/DowningStrategy/DowningClusteredTests.swift b/Tests/DistributedActorsTests/Cluster/DowningStrategy/DowningClusteredTests.swift index d9f55fa77..71d1b7876 100644 --- a/Tests/DistributedActorsTests/Cluster/DowningStrategy/DowningClusteredTests.swift +++ b/Tests/DistributedActorsTests/Cluster/DowningStrategy/DowningClusteredTests.swift @@ -279,7 +279,7 @@ final class DowningClusteredTests: ClusteredActorSystemsXCTestCase { for remainingNode in nodes { let probe = probes[remainingNode.cluster.node]! - let events = try probe.fishFor(Cluster.MembershipChange.self, within: .seconds(60), expectedDownMemberEventsFishing(on: remainingNode)) + let events = try probe.fishFor(Cluster.MembershipChange.self, within: .seconds(120), expectedDownMemberEventsFishing(on: remainingNode)) events.shouldContain(where: { change in change.toStatus.isDown && (change.fromStatus == .joining || change.fromStatus == .up) }) for expectedDownNode in nodesToDown { diff --git a/Tests/DistributedActorsTests/Cluster/GossipSeenTableTests.swift b/Tests/DistributedActorsTests/Cluster/GossipSeenTableTests.swift index 19b0b7a87..9560db733 100644 --- a/Tests/DistributedActorsTests/Cluster/GossipSeenTableTests.swift +++ b/Tests/DistributedActorsTests/Cluster/GossipSeenTableTests.swift @@ -19,7 +19,7 @@ import XCTest /// Tests of just the datatype final class GossipSeenTableTests: XCTestCase { - typealias SeenTable = Cluster.Gossip.SeenTable + typealias SeenTable = Cluster.MembershipGossip.SeenTable var nodeA: UniqueNode! var nodeB: UniqueNode! @@ -37,13 +37,13 @@ final class GossipSeenTableTests: XCTestCase { } func test_seenTable_compare_concurrent_eachOtherDontKnown() { - let table = Cluster.Gossip.SeenTable.parse( + let table = Cluster.MembershipGossip.SeenTable.parse( """ A: A@1 """, nodes: self.allNodes ) - let incoming = Cluster.Gossip.SeenTable.parse( + let incoming = Cluster.MembershipGossip.SeenTable.parse( """ B: B@1 """, nodes: self.allNodes @@ -61,7 +61,7 @@ final class GossipSeenTableTests: XCTestCase { // MARK: increments func test_incrementVersion() { - var table = Cluster.Gossip.SeenTable(myselfNode: self.nodeA, version: .init()) + var table = Cluster.MembershipGossip.SeenTable(myselfNode: self.nodeA, version: .init()) table.version(at: self.nodeA).shouldEqual(VersionVector.parse("", nodes: self.allNodes)) table.incrementVersion(owner: self.nodeA, at: self.nodeA) @@ -83,13 +83,13 @@ final class GossipSeenTableTests: XCTestCase { // MARK: merge func test_seenTable_merge_notYetSeenInformation() { - var table = Cluster.Gossip.SeenTable.parse( + var table = Cluster.MembershipGossip.SeenTable.parse( """ A: A:1 """, nodes: self.allNodes ) - let incoming = Cluster.Gossip.SeenTable.parse( + let incoming = Cluster.MembershipGossip.SeenTable.parse( """ B: B:2 """, nodes: self.allNodes @@ -106,12 +106,12 @@ final class GossipSeenTableTests: XCTestCase { func test_seenTable_merge_sameInformation() { // a situation in which the two nodes have converged, so their versions are .same - var table = Cluster.Gossip.SeenTable(myselfNode: self.nodeA, version: .init()) + var table = Cluster.MembershipGossip.SeenTable(myselfNode: self.nodeA, version: .init()) table.incrementVersion(owner: self.nodeA, at: self.nodeA) // A observed: A:1 table.incrementVersion(owner: self.nodeA, at: self.nodeB) // A observed: A:1 B:1 table.incrementVersion(owner: self.nodeA, at: self.nodeB) // A observed: A:1 B:2 - var incoming = Cluster.Gossip(ownerNode: self.nodeB) // B observed: + var incoming = Cluster.MembershipGossip(ownerNode: self.nodeB) // B observed: incoming.incrementOwnerVersion() // B observed: B:1 incoming.incrementOwnerVersion() // B observed: B:2 incoming.seen.incrementVersion(owner: self.nodeB, at: self.nodeA) // B observed: A:1 B:2 @@ -125,10 +125,10 @@ final class GossipSeenTableTests: XCTestCase { func test_seenTable_merge_aheadInformation() { // the incoming gossip is "ahead" and has some more information - var table = Cluster.Gossip.SeenTable(myselfNode: self.nodeA, version: .init()) + var table = Cluster.MembershipGossip.SeenTable(myselfNode: self.nodeA, version: .init()) table.incrementVersion(owner: self.nodeA, at: self.nodeA) // A observed: A:1 - var incoming = Cluster.Gossip(ownerNode: self.nodeB) // B observed: + var incoming = Cluster.MembershipGossip(ownerNode: self.nodeB) // B observed: incoming.incrementOwnerVersion() // B observed: B:1 incoming.incrementOwnerVersion() // B observed: B:2 incoming.seen.incrementVersion(owner: self.nodeB, at: self.nodeA) // B observed: A:1 B:2 @@ -142,12 +142,12 @@ final class GossipSeenTableTests: XCTestCase { func test_seenTable_merge_behindInformation() { // the incoming gossip is "behind" - var table = Cluster.Gossip.SeenTable(myselfNode: self.nodeA, version: .init()) + var table = Cluster.MembershipGossip.SeenTable(myselfNode: self.nodeA, version: .init()) table.incrementVersion(owner: self.nodeA, at: self.nodeA) // A observed: A:1 table.incrementVersion(owner: self.nodeA, at: self.nodeB) // A observed: A:1 B:1 table.incrementVersion(owner: self.nodeA, at: self.nodeB) // A observed: A:1 B:2 - var incoming = Cluster.Gossip(ownerNode: self.nodeB) // B observed: + var incoming = Cluster.MembershipGossip(ownerNode: self.nodeB) // B observed: incoming.incrementOwnerVersion() // B observed: B:1 incoming.incrementOwnerVersion() // B observed: B:2 @@ -160,7 +160,7 @@ final class GossipSeenTableTests: XCTestCase { func test_seenTable_merge_concurrentInformation() { // the incoming gossip is "concurrent" - var table = Cluster.Gossip.SeenTable(myselfNode: self.nodeA, version: .init()) + var table = Cluster.MembershipGossip.SeenTable(myselfNode: self.nodeA, version: .init()) table.incrementVersion(owner: self.nodeA, at: self.nodeA) // A observed: A:1 table.incrementVersion(owner: self.nodeA, at: self.nodeB) // A observed: A:1 B:1 table.incrementVersion(owner: self.nodeA, at: self.nodeB) // A observed: A:1 B:2 @@ -170,7 +170,7 @@ final class GossipSeenTableTests: XCTestCase { table.incrementVersion(owner: self.nodeB, at: self.nodeB) // B observed: B:3 // in reality S is quite more far ahead, already at t=4 - var incoming = Cluster.Gossip(ownerNode: self.nodeB) // B observed + var incoming = Cluster.MembershipGossip(ownerNode: self.nodeB) // B observed incoming.incrementOwnerVersion() // B observed: B:1 incoming.incrementOwnerVersion() // B observed: B:2 incoming.incrementOwnerVersion() // B observed: B:3 @@ -185,13 +185,13 @@ final class GossipSeenTableTests: XCTestCase { func test_seenTable_merge_concurrentInformation_unknownMember() { // the incoming gossip is "concurrent", and has a table entry for a node we don't know - var table = Cluster.Gossip.SeenTable.parse( + var table = Cluster.MembershipGossip.SeenTable.parse( """ A: A:4 """, nodes: self.allNodes ) - let incoming = Cluster.Gossip.SeenTable.parse( + let incoming = Cluster.MembershipGossip.SeenTable.parse( """ A: A:1 B: B:2 C:1 @@ -210,7 +210,7 @@ final class GossipSeenTableTests: XCTestCase { // MARK: Prune func test_prune_removeNodeFromSeenTable() { - var table = Cluster.Gossip.SeenTable(myselfNode: self.nodeA, version: .init()) + var table = Cluster.MembershipGossip.SeenTable(myselfNode: self.nodeA, version: .init()) table.incrementVersion(owner: self.nodeA, at: self.nodeA) table.incrementVersion(owner: self.nodeA, at: self.nodeC) diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift index 4a21b1c4f..6a706a8a2 100644 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift @@ -57,13 +57,13 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas return logic } - var gossips: [Cluster.Gossip] { + var gossips: [Cluster.MembershipGossip] { self.logics.map { $0.latestGossip } } - private func makeLogic(_ system: ActorSystem, _ probe: ActorTestProbe) -> MembershipGossipLogic { + private func makeLogic(_ system: ActorSystem, _ probe: ActorTestProbe) -> MembershipGossipLogic { MembershipGossipLogic( - GossipLogicContext( + GossipLogicContext( ownerContext: self.testKit(system).makeFakeContext(), gossipIdentifier: StringGossipIdentifier("membership") ), @@ -93,16 +93,16 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas runs: 10, setUpPeers: { () in [ - Cluster.Gossip.parse(initialGossipState, owner: systemA.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialGossipState, owner: systemB.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialGossipState, owner: systemC.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialGossipState, owner: systemA.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialGossipState, owner: systemB.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialGossipState, owner: systemC.cluster.node, nodes: self.nodes), ] }, updateLogic: { _ in let logicA: MembershipGossipLogic = self.logic("A") // We simulate that `A` noticed it's the leader and moved `B` and `C` .up - logicA.localGossipUpdate(gossip: Cluster.Gossip.parse( + logicA.receiveLocalGossipUpdate(Cluster.MembershipGossip.parse( """ A.up B.up C.up A: A@5 B@3 C@3 @@ -171,24 +171,24 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas runs: 1, setUpPeers: { () in [ - Cluster.Gossip.parse(initialFewGossip, owner: systemA.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialFewGossip, owner: systemB.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialFewGossip, owner: systemC.cluster.node, nodes: self.nodes), - - Cluster.Gossip.parse(initialNewGossip, owner: systemD.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialNewGossip, owner: systemE.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialNewGossip, owner: systemF.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialNewGossip, owner: systemG.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialNewGossip, owner: systemH.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialNewGossip, owner: systemI.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialNewGossip, owner: systemJ.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialFewGossip, owner: systemA.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialFewGossip, owner: systemB.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialFewGossip, owner: systemC.cluster.node, nodes: self.nodes), + + Cluster.MembershipGossip.parse(initialNewGossip, owner: systemD.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialNewGossip, owner: systemE.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialNewGossip, owner: systemF.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialNewGossip, owner: systemG.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialNewGossip, owner: systemH.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialNewGossip, owner: systemI.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialNewGossip, owner: systemJ.cluster.node, nodes: self.nodes), ] }, updateLogic: { _ in let logicA: MembershipGossipLogic = self.logic("A") let logicD: MembershipGossipLogic = self.logic("D") - logicA.localGossipUpdate(gossip: Cluster.Gossip.parse( + logicA.receiveLocalGossipUpdate(Cluster.MembershipGossip.parse( """ A.up B.up C.up D.up E.up F.up G.up H.up I.up J.up A: A@20 B@16 C@16 D@16 E@16 F@16 G@16 H@16 I@16 J@16 @@ -206,7 +206,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas )) // they're trying to join - logicD.localGossipUpdate(gossip: Cluster.Gossip.parse( + logicD.receiveLocalGossipUpdate(Cluster.MembershipGossip.parse( """ A.up B.up C.up D.joining E.joining F.joining G.joining H.joining I.joining J.joining A: A@11 B@16 C@16 D@9 E@13 F@13 G@13 H@13 I@13 J@13 @@ -254,16 +254,16 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas runs: 10, setUpPeers: { () in [ - Cluster.Gossip.parse(initialGossipState, owner: systemA.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialGossipState, owner: systemB.cluster.node, nodes: self.nodes), - Cluster.Gossip.parse(initialGossipState, owner: systemC.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialGossipState, owner: systemA.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialGossipState, owner: systemB.cluster.node, nodes: self.nodes), + Cluster.MembershipGossip.parse(initialGossipState, owner: systemC.cluster.node, nodes: self.nodes), ] }, updateLogic: { _ in let logicA: MembershipGossipLogic = self.logic("A") // We simulate that `A` noticed it's the leader and moved `B` and `C` .up - logicA.localGossipUpdate(gossip: Cluster.Gossip.parse( + logicA.receiveLocalGossipUpdate(Cluster.MembershipGossip.parse( """ A.up B.up C.up A: A@5 B@3 C@3 @@ -290,7 +290,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas func gossipSimulationTest( runs: Int, - setUpPeers: () -> [Cluster.Gossip], + setUpPeers: () -> [Cluster.MembershipGossip], updateLogic: ([MembershipGossipLogic]) -> Void, stopRunWhen: ([MembershipGossipLogic], GossipSimulationResults) -> Bool, assert: (GossipSimulationResults) -> Void @@ -305,8 +305,8 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas ) let initialGossips = setUpPeers() - self.mockPeers = try! self.systems.map { system -> ActorRef.Message> in - let ref: ActorRef.Message> = + self.mockPeers = try! self.systems.map { system -> ActorRef.Message> in + let ref: ActorRef.Message> = try system.spawn("peer", .receiveMessage { _ in .same }) return self.systems.first!._resolveKnownRemote(ref, onRemoteSystem: system) }.map { $0.asAddressable() } @@ -318,13 +318,13 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas // initialize with user provided gossips self.logics = initialGossips.map { initialGossip in let system = self.system(initialGossip.owner.node.systemName) - let probe = self.testKit(system).spawnTestProbe(expecting: Cluster.Gossip.self) + let probe = self.testKit(system).spawnTestProbe(expecting: Cluster.MembershipGossip.self) let logic = self.makeLogic(system, probe) - logic.localGossipUpdate(gossip: initialGossip) + logic.receiveLocalGossipUpdate(initialGossip) return logic } - func allConverged(gossips: [Cluster.Gossip]) -> Bool { + func allConverged(gossips: [Cluster.MembershipGossip]) -> Bool { var allSatisfied = true // on purpose not via .allSatisfy() since we want to print status of each logic for g in gossips.sorted(by: { $0.owner.node.systemName < $1.owner.node.systemName }) { let converged = g.converged() @@ -360,7 +360,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas ]) let targetLogic = self.selectLogic(targetPeer) - let maybeAck = targetLogic.receiveGossip(gossip: gossip, from: self.peer(logic)) + let maybeAck = targetLogic.receiveGossip(gossip, from: self.peer(logic)) log.notice("updated [\(targetPeer.address.node!.node.systemName)]", metadata: [ "gossip": Logger.MetadataValue.pretty(targetLogic.latestGossip), ]) @@ -369,7 +369,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas log.notice(" \(logic.nodeName) <- \(targetPeer.address.node!.node.systemName) (ack)", metadata: [ "ack": Logger.MetadataValue.pretty(ack), ]) - logic.receiveAcknowledgement(from: self.peer(targetLogic), acknowledgement: ack, confirmsDeliveryOf: gossip) + logic.receiveAcknowledgement(ack, from: self.peer(targetLogic), confirming: gossip) } } else { diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipTests.swift index 0e06cfb3b..48f41f061 100644 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipTests.swift +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipTests.swift @@ -39,14 +39,14 @@ final class MembershipGossipTests: XCTestCase { // MARK: Merging gossips func test_mergeForward_incomingGossip_firstGossipFromOtherNode() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.joining A: A:1 """, owner: self.nodeA, nodes: self.allNodes ) - let incoming = Cluster.Gossip.parse( + let incoming = Cluster.MembershipGossip.parse( """ B.joining B: B:1 @@ -60,7 +60,7 @@ final class MembershipGossipTests: XCTestCase { ) gossip.shouldEqual( - Cluster.Gossip.parse( + Cluster.MembershipGossip.parse( """ A.joining B.joining A: A:1 B:1 @@ -71,14 +71,14 @@ final class MembershipGossipTests: XCTestCase { } func test_mergeForward_incomingGossip_firstGossipFromOtherNodes() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.joining A: A:1 """, owner: self.nodeA, nodes: self.allNodes ) - let incoming = Cluster.Gossip.parse( + let incoming = Cluster.MembershipGossip.parse( """ B.joining C.joining B: B:1 C:1 @@ -97,7 +97,7 @@ final class MembershipGossipTests: XCTestCase { ) ) - let expected = Cluster.Gossip.parse( + let expected = Cluster.MembershipGossip.parse( """ A.joining B.joining C.joining A: A:1 B:1 C:1 @@ -115,19 +115,19 @@ final class MembershipGossipTests: XCTestCase { } func test_mergeForward_incomingGossip_sameVersions() { - var gossip = Cluster.Gossip(ownerNode: self.nodeA) + var gossip = Cluster.MembershipGossip(ownerNode: self.nodeA) _ = gossip.membership.join(self.nodeA) gossip.seen.incrementVersion(owner: self.nodeB, at: self.nodeA) // v: myself:1, second:1 _ = gossip.membership.join(self.nodeB) // myself:joining, second:joining - let gossipFromSecond = Cluster.Gossip(ownerNode: self.nodeB) + let gossipFromSecond = Cluster.MembershipGossip(ownerNode: self.nodeB) let directive = gossip.mergeForward(incoming: gossipFromSecond) directive.effectiveChanges.shouldEqual([]) } func test_mergeForward_incomingGossip_fromFourth_onlyKnowsAboutItself() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.joining B.joining B.joining A: A@1 B@1 C@1 @@ -135,7 +135,7 @@ final class MembershipGossipTests: XCTestCase { ) // only knows about fourth, while myGossip has first, second and third - let incomingGossip = Cluster.Gossip.parse( + let incomingGossip = Cluster.MembershipGossip.parse( """ D.joining D: D@1 @@ -150,7 +150,7 @@ final class MembershipGossipTests: XCTestCase { [Cluster.MembershipChange(node: self.fourthNode, fromStatus: nil, toStatus: .joining)] ) gossip.shouldEqual( - Cluster.Gossip.parse( + Cluster.MembershipGossip.parse( """ A.joining B.joining C.joining A: A@1 B@1 C@1 D@1 @@ -161,7 +161,7 @@ final class MembershipGossipTests: XCTestCase { } func test_mergeForward_incomingGossip_localHasRemoved_incomingHasOldViewWithDownNode() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.up B.down C.up A: A@5 B@5 C@6 @@ -177,7 +177,7 @@ final class MembershipGossipTests: XCTestCase { _ = gossip.pruneMember(removedMember) gossip.incrementOwnerVersion() - let incomingOldGossip = Cluster.Gossip.parse( + let incomingOldGossip = Cluster.MembershipGossip.parse( """ A.up B.down C.up A: A@5 B@5 C@6 @@ -203,7 +203,7 @@ final class MembershipGossipTests: XCTestCase { } func test_mergeForward_incomingGossip_concurrent_leaderDisagreement() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.up B.joining [leader:A] A: A@5 B@5 @@ -216,7 +216,7 @@ final class MembershipGossipTests: XCTestCase { // once the nodes talk to each other again, they will run leader election and resolve the double leader situation // until that happens, the two leaders indeed remain as-is -- as the membership itself is not the right place to resolve // who shall be leader - let incomingGossip = Cluster.Gossip.parse( + let incomingGossip = Cluster.MembershipGossip.parse( """ A.up B.joining C.up [leader:B] B: B@2 C@1 @@ -242,7 +242,7 @@ final class MembershipGossipTests: XCTestCase { ] ) - let expected = Cluster.Gossip.parse( + let expected = Cluster.MembershipGossip.parse( """ A.up B.joining C.up [leader:A] A: A:5 B:5 C:9 @@ -257,14 +257,14 @@ final class MembershipGossipTests: XCTestCase { } func test_mergeForward_incomingGossip_concurrent_simple() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.up B.joining A: A@4 """, owner: self.nodeA, nodes: self.allNodes ) - let concurrent = Cluster.Gossip.parse( + let concurrent = Cluster.MembershipGossip.parse( """ A.joining B.joining B: B@2 @@ -276,7 +276,7 @@ final class MembershipGossipTests: XCTestCase { gossip.owner.shouldEqual(self.nodeA) directive.effectiveChanges.count.shouldEqual(0) gossip.shouldEqual( - Cluster.Gossip.parse( + Cluster.MembershipGossip.parse( """ A.up B.joining A: A@4 B@2 @@ -287,7 +287,7 @@ final class MembershipGossipTests: XCTestCase { } func test_mergeForward_incomingGossip_hasNewNode() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.up A: A@5 @@ -310,7 +310,7 @@ final class MembershipGossipTests: XCTestCase { } func test_mergeForward_removal_incomingGossip_isAhead_hasRemovedNodeKnownToBeDown() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.up B.down C.up [leader:A] A: A@5 B@5 C@6 @@ -320,7 +320,7 @@ final class MembershipGossipTests: XCTestCase { owner: self.nodeA, nodes: self.allNodes ) - let incomingGossip = Cluster.Gossip.parse( + let incomingGossip = Cluster.MembershipGossip.parse( """ A.up C.up A: A@5 C@6 @@ -340,7 +340,7 @@ final class MembershipGossipTests: XCTestCase { ) gossip.shouldEqual( - Cluster.Gossip.parse( + Cluster.MembershipGossip.parse( """ A.up C.up [leader:A] A: A@5 C@7 @@ -351,7 +351,7 @@ final class MembershipGossipTests: XCTestCase { } func test_mergeForward_incomingGossip_removal_isAhead_hasMyNodeRemoved_thusWeKeepItAsRemoved() { - var gossip = Cluster.Gossip.parse( + var gossip = Cluster.MembershipGossip.parse( """ A.up B.down C.up A: A@5 B@5 C@6 @@ -361,7 +361,7 @@ final class MembershipGossipTests: XCTestCase { owner: self.nodeB, nodes: self.allNodes ) - let incomingGossip = Cluster.Gossip.parse( + let incomingGossip = Cluster.MembershipGossip.parse( """ A.up C.up A: A@5 C@6 @@ -380,7 +380,7 @@ final class MembershipGossipTests: XCTestCase { // we MIGHT receive a removal of "our node" however we must never apply such change! // we know we are `.down` and that's the most "we" will ever perceive ourselves as -- i.e. removed is only for "others". - let expected = Cluster.Gossip.parse( + let expected = Cluster.MembershipGossip.parse( """ A.up B.removed C.up A: A@5 B@5 C@6 @@ -398,7 +398,7 @@ final class MembershipGossipTests: XCTestCase { // MARK: Convergence func test_converged_shouldBeTrue_forNoMembers() { - var gossip = Cluster.Gossip(ownerNode: self.nodeA) + var gossip = Cluster.MembershipGossip(ownerNode: self.nodeA) _ = gossip.membership.join(self.nodeA) gossip.converged().shouldBeTrue() @@ -407,7 +407,7 @@ final class MembershipGossipTests: XCTestCase { } func test_converged_amongUpMembers() { - var gossip = Cluster.Gossip(ownerNode: self.nodeA) + var gossip = Cluster.MembershipGossip(ownerNode: self.nodeA) _ = gossip.membership.join(self.nodeA) _ = gossip.membership.mark(self.nodeA, as: .up) @@ -451,7 +451,7 @@ final class MembershipGossipTests: XCTestCase { } func test_converged_othersAreOnlyDown() { - let gossip = Cluster.Gossip.parse( + let gossip = Cluster.MembershipGossip.parse( """ A.up B.down A: A@8 B@5 @@ -466,7 +466,7 @@ final class MembershipGossipTests: XCTestCase { // FIXME: we should not need .joining nodes to participate on convergence() func fixme_converged_joiningOrDownMembersDoNotCount() { - var gossip = Cluster.Gossip(ownerNode: self.nodeA) + var gossip = Cluster.MembershipGossip(ownerNode: self.nodeA) _ = gossip.membership.join(self.nodeA) _ = gossip.membership.join(self.nodeB) @@ -518,8 +518,8 @@ final class MembershipGossipTests: XCTestCase { } func test_gossip_eventuallyConverges() { - func makeRandomGossip(owner node: UniqueNode) -> Cluster.Gossip { - var gossip = Cluster.Gossip(ownerNode: node) + func makeRandomGossip(owner node: UniqueNode) -> Cluster.MembershipGossip { + var gossip = Cluster.MembershipGossip(ownerNode: node) _ = gossip.membership.join(node) _ = gossip.membership.mark(node, as: .joining) var vv = VersionVector() diff --git a/Tests/DistributedActorsTests/Cluster/Protobuf/Membership+SerializationTests.swift b/Tests/DistributedActorsTests/Cluster/Protobuf/Membership+SerializationTests.swift index 696a8c5fa..7de484e76 100644 --- a/Tests/DistributedActorsTests/Cluster/Protobuf/Membership+SerializationTests.swift +++ b/Tests/DistributedActorsTests/Cluster/Protobuf/Membership+SerializationTests.swift @@ -58,7 +58,7 @@ final class MembershipSerializationTests: ActorSystemXCTestCase { } let nodes = members.map { $0.node } - let gossip = Cluster.Gossip.parse( + let gossip = Cluster.MembershipGossip.parse( """ 1.joining 2.joining 3.joining 4.up 5.up 6.up 7.up 8.up 9.down 10.down 11.up 12.up 13.up 14.up 15.up 1: 1:4 2:4 3:4 4:6 5:7 6:7 7:8 8:8 9:12 10:12 11:8 12:8 13:8 14:9 15:6 @@ -84,7 +84,7 @@ final class MembershipSerializationTests: ActorSystemXCTestCase { serialized.manifest.serializerID.shouldEqual(Serialization.SerializerID.protobufRepresentable) serialized.buffer.count.shouldEqual(2105) - let back = try system.serialization.deserialize(as: Cluster.Gossip.self, from: serialized) + let back = try system.serialization.deserialize(as: Cluster.MembershipGossip.self, from: serialized) "\(pretty: back)".shouldStartWith(prefix: "\(pretty: gossip)") // nicer human readable error back.shouldEqual(gossip) // the actual sanity check } diff --git a/Tests/DistributedActorsTests/Cluster/TestExtensions+MembershipDSL.swift b/Tests/DistributedActorsTests/Cluster/TestExtensions+MembershipDSL.swift index 1f75126f2..90f9410b5 100644 --- a/Tests/DistributedActorsTests/Cluster/TestExtensions+MembershipDSL.swift +++ b/Tests/DistributedActorsTests/Cluster/TestExtensions+MembershipDSL.swift @@ -19,21 +19,21 @@ import NIO // ==== ---------------------------------------------------------------------------------------------------------------- // MARK: Membership Testing DSL -extension Cluster.Gossip { +extension Cluster.MembershipGossip { /// First line is Membership DSL, followed by lines of the SeenTable DSL - internal static func parse(_ dsl: String, owner: UniqueNode, nodes: [UniqueNode]) -> Cluster.Gossip { + internal static func parse(_ dsl: String, owner: UniqueNode, nodes: [UniqueNode]) -> Cluster.MembershipGossip { let dslLines = dsl.split(separator: "\n") - var gossip = Cluster.Gossip(ownerNode: owner) + var gossip = Cluster.MembershipGossip(ownerNode: owner) gossip.membership = Cluster.Membership.parse(String(dslLines.first!), nodes: nodes) - gossip.seen = Cluster.Gossip.SeenTable.parse(dslLines.dropFirst().joined(separator: "\n"), nodes: nodes) + gossip.seen = Cluster.MembershipGossip.SeenTable.parse(dslLines.dropFirst().joined(separator: "\n"), nodes: nodes) return gossip } } -extension Cluster.Gossip.SeenTable { +extension Cluster.MembershipGossip.SeenTable { /// Express seen tables using a DSL /// Syntax: each line: `: @*` - internal static func parse(_ dslString: String, nodes: [UniqueNode], file: StaticString = #file, line: UInt = #line) -> Cluster.Gossip.SeenTable { + internal static func parse(_ dslString: String, nodes: [UniqueNode], file: StaticString = #file, line: UInt = #line) -> Cluster.MembershipGossip.SeenTable { let lines = dslString.split(separator: "\n") func nodeById(id: String.SubSequence) -> UniqueNode { if let found = nodes.first(where: { $0.node.systemName.contains(id) }) { @@ -43,7 +43,7 @@ extension Cluster.Gossip.SeenTable { } } - var table = Cluster.Gossip.SeenTable() + var table = Cluster.MembershipGossip.SeenTable() for line in lines { let elements = line.split(separator: " ") diff --git a/Tests/DistributedActorsTests/Cluster/TestExtensions.swift b/Tests/DistributedActorsTests/Cluster/TestExtensions.swift index 7ddcdd916..3ced7076c 100644 --- a/Tests/DistributedActorsTests/Cluster/TestExtensions.swift +++ b/Tests/DistributedActorsTests/Cluster/TestExtensions.swift @@ -38,7 +38,7 @@ extension ClusterShellState { settings: settings, channel: EmbeddedChannel(), events: EventStream(ref: ActorRef(.deadLetters(.init(log, address: ._deadLetters, system: nil)))), - gossipControl: GossipControl(ActorRef(.deadLetters(.init(log, address: ._deadLetters, system: nil)))), + gossiperControl: GossiperControl(ActorRef(.deadLetters(.init(log, address: ._deadLetters, system: nil)))), log: log ) } diff --git a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift b/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift index ddb54d04a..0a3ec1288 100644 --- a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift +++ b/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift @@ -29,7 +29,7 @@ final class GossipShellTests: ActorSystemXCTestCase { makeLogic: { _ in InspectOfferedPeersTestGossipLogic(offeredPeersProbe: p.ref) } ) - let peerBehavior: Behavior.Message> = .receiveMessage { msg in + let peerBehavior: Behavior.Message> = .receiveMessage { msg in if "\(msg)".contains("stop") { return .stop } else { return .same } } let first = try self.system.spawn("first", peerBehavior) @@ -49,7 +49,7 @@ final class GossipShellTests: ActorSystemXCTestCase { } struct InspectOfferedPeersTestGossipLogic: GossipLogic { - struct Envelope: GossipEnvelopeProtocol { + struct Gossip: GossipEnvelopeProtocol { let metadata: String let payload: String @@ -71,16 +71,16 @@ final class GossipShellTests: ActorSystemXCTestCase { return [] } - func makePayload(target: AddressableActorRef) -> Envelope? { + func makePayload(target: AddressableActorRef) -> Gossip? { nil } - func receiveAcknowledgement(from peer: AddressableActorRef, acknowledgement: Acknowledgement, confirmsDeliveryOf envelope: Envelope) {} + func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming envelope: Gossip) {} - func receiveGossip(gossip: Envelope, from peer: AddressableActorRef) -> Acknowledgement? { + func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? { nil } - func localGossipUpdate(gossip: Envelope) {} + func receiveLocalGossipUpdate(_ gossip: Gossip) {} } } From f8027176aa08fa49490585199c3433f583065ec8 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Wed, 24 Jun 2020 14:58:39 +0900 Subject: [PATCH 11/15] =gossip cleanups and introduce configurable gossip style; warn when unexpected ACks are required --- .../DistributedActors/CRDT/CRDT+Gossip.swift | 8 +- .../CRDT/CRDT+Replication.swift | 4 + .../CRDT/CRDT+ReplicatorShell.swift | 5 +- .../Cluster/ClusterSettings.swift | 6 + .../Cluster/ClusterShell.swift | 8 +- .../Cluster+MembershipGossip.swift | 13 --- .../Cluster+MembershipGossipLogic.swift | 11 +- .../Gossip/Gossip+Logic.swift | 80 +++++++------ .../Gossip/Gossip+Serialization.swift | 7 +- .../Gossip/Gossip+Settings.swift | 26 ++++- .../Gossip/Gossip+Shell.swift | 109 ++++++++++++------ .../Gossip/PeerSelection.swift | 34 ------ ...MembershipGossipLogicSimulationTests.swift | 4 +- .../Gossip/GossipShellTests.swift | 101 ++++++++++++++-- 14 files changed, 263 insertions(+), 153 deletions(-) diff --git a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift index dffe5f3c8..0a6c4ab6b 100644 --- a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift +++ b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift @@ -55,7 +55,7 @@ extension CRDT { // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Spreading gossip - func selectPeers(peers: [AddressableActorRef]) -> [AddressableActorRef] { + func selectPeers(_ peers: [AddressableActorRef]) -> [AddressableActorRef] { // how many peers we select in each gossip round, // we could for example be dynamic and notice if we have 10+ nodes, we pick 2 members to speed up the dissemination etc. let n = 1 @@ -80,8 +80,8 @@ extension CRDT { self.latest } - func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming envelope: CRDT.Gossip) { - guard (self.latest.map { $0.payload.equalState(to: envelope.payload) } ?? false) else { + func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming gossip: CRDT.Gossip) { + guard (self.latest.map { $0.payload.equalState(to: gossip.payload) } ?? false) else { // received an ack for something, however it's not the "latest" anymore, so we need to gossip to target anyway return } @@ -180,7 +180,7 @@ extension CRDT.Identity: GossipIdentifier { extension CRDT { /// The gossip to be spread about a specific CRDT (identity). - struct Gossip: GossipEnvelopeProtocol, CustomStringConvertible, CustomPrettyStringConvertible { + struct Gossip: Codable, CustomStringConvertible, CustomPrettyStringConvertible { struct Metadata: Codable {} // FIXME: remove, seems we dont need metadata explicitly here typealias Payload = StateBasedCRDT diff --git a/Sources/DistributedActors/CRDT/CRDT+Replication.swift b/Sources/DistributedActors/CRDT/CRDT+Replication.swift index 42a62a014..33a2369a7 100644 --- a/Sources/DistributedActors/CRDT/CRDT+Replication.swift +++ b/Sources/DistributedActors/CRDT/CRDT+Replication.swift @@ -278,6 +278,10 @@ extension CRDT { /// during this round public var gossipInterval: TimeAmount = .seconds(2) + /// Timeout used when asking another peer when spreading gossip. + /// Timeouts are logged, but by themselves not "errors", as we still eventually may be able to spread the payload to given peer. + public var gossipAcknowledgementTimeout: TimeAmount = .milliseconds(500) + /// Adds a random factor to the gossip interval, which is useful to avoid an entire cluster ticking "synchronously" /// at the same time, causing spikes in gossip traffic (as all nodes decide to gossip in the same second). /// diff --git a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift index f90d96b55..3d326b825 100644 --- a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift +++ b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift @@ -75,8 +75,9 @@ extension CRDT.Replicator { context, name: "gossip", settings: Gossiper.Settings( - gossipInterval: self.settings.gossipInterval, - gossipIntervalRandomFactor: self.settings.gossipIntervalRandomFactor, + interval: self.settings.gossipInterval, + intervalRandomFactor: self.settings.gossipIntervalRandomFactor, + style: .acknowledged(timeout: .seconds(1)), peerDiscovery: .fromReceptionistListing(id: "crdt-gossip-replicator") ), makeLogic: { logicContext in diff --git a/Sources/DistributedActors/Cluster/ClusterSettings.swift b/Sources/DistributedActors/Cluster/ClusterSettings.swift index b2de9c659..7995b0ed1 100644 --- a/Sources/DistributedActors/Cluster/ClusterSettings.swift +++ b/Sources/DistributedActors/Cluster/ClusterSettings.swift @@ -110,6 +110,12 @@ public struct ClusterSettings { public var membershipGossipInterval: TimeAmount = .seconds(1) + // since we talk to many peers one by one; even as we proceed to the next round after `membershipGossipInterval` + // it is fine if we get a reply from the previously gossiped to peer after same or similar timeout. No rush about it. + // + // A missing ACK is not terminal, may happen, and we'll then gossip with that peer again (e.g. if it ha had some form of network trouble for a moment). + public var membershipGossipAcknowledgementTimeout: TimeAmount = .seconds(1) + public var membershipGossipIntervalRandomFactor: Double = 0.2 // ==== ------------------------------------------------------------------------------------------------------------ diff --git a/Sources/DistributedActors/Cluster/ClusterShell.swift b/Sources/DistributedActors/Cluster/ClusterShell.swift index f4fe58b87..1962dca2b 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell.swift @@ -422,19 +422,19 @@ extension ClusterShell { return context.awaitResultThrowing(of: chanElf, timeout: clusterSettings.bindTimeout) { (chan: Channel) in context.log.info("Bound to \(chan.localAddress.map { $0.description } ?? "")") - // TODO: Membership.Gossip? let gossiperControl: GossiperControl = try Gossiper.start( context, name: "\(ActorPath._clusterGossip.name)", - props: ._wellKnown, settings: .init( - gossipInterval: clusterSettings.membershipGossipInterval, - gossipIntervalRandomFactor: clusterSettings.membershipGossipIntervalRandomFactor, + interval: clusterSettings.membershipGossipInterval, + intervalRandomFactor: clusterSettings.membershipGossipIntervalRandomFactor, + style: .acknowledged(timeout: clusterSettings.membershipGossipInterval), peerDiscovery: .onClusterMember(atLeast: .joining, resolve: { member in let resolveContext = ResolveContext.Message>(address: ._clusterGossip(on: member.node), system: context.system) return context.system._resolve(context: resolveContext).asAddressable() }) ), + props: ._wellKnown, makeLogic: { MembershipGossipLogic( $0, diff --git a/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift index f2da92902..a0c061cd4 100644 --- a/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift +++ b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift @@ -160,19 +160,6 @@ extension Cluster { } -extension Cluster.MembershipGossip: GossipEnvelopeProtocol { - typealias Metadata = SeenTable - typealias Payload = Self - - var metadata: Metadata { - self.seen - } - - var payload: Payload { - self - } -} - extension Cluster.MembershipGossip: CustomPrettyStringConvertible {} // ==== ---------------------------------------------------------------------------------------------------------------- diff --git a/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossipLogic.swift b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossipLogic.swift index 95260e8ec..578a1e996 100644 --- a/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossipLogic.swift +++ b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossipLogic.swift @@ -53,13 +53,12 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Spreading gossip - // TODO: implement better, only peers which are "behind" - func selectPeers(peers _peers: [AddressableActorRef]) -> [AddressableActorRef] { + func selectPeers(_ peers: [AddressableActorRef]) -> [AddressableActorRef] { // how many peers we select in each gossip round, // we could for example be dynamic and notice if we have 10+ nodes, we pick 2 members to speed up the dissemination etc. let n = 1 - self.updateActivePeers(peers: _peers) + self.updateActivePeers(peers) var selectedPeers: [AddressableActorRef] = [] selectedPeers.reserveCapacity(min(n, self.peers.count)) @@ -73,7 +72,7 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { return selectedPeers } - private func updateActivePeers(peers: [AddressableActorRef]) { + private func updateActivePeers(_ peers: [AddressableActorRef]) { if let changed = Self.peersChanged(known: self.peers, current: peers) { // 1) remove any peers which are no longer active // - from the peers list @@ -205,8 +204,8 @@ final class MembershipGossipLogic: GossipLogic, CustomStringConvertible { let MembershipGossipIdentifier: StringGossipIdentifier = "membership" -extension GossiperControl where GossipEnvelope == Cluster.MembershipGossip { - func update(payload: GossipEnvelope) { +extension GossiperControl where Gossip == Cluster.MembershipGossip { + func update(payload: Gossip) { self.update(MembershipGossipIdentifier, payload: payload) } diff --git a/Sources/DistributedActors/Gossip/Gossip+Logic.swift b/Sources/DistributedActors/Gossip/Gossip+Logic.swift index 2e7d64fb2..3820acd7e 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Logic.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Logic.swift @@ -38,13 +38,11 @@ import Logging /// for a nice overview of the general concepts involved in gossip algorithms. /// - SeeAlso: `Cluster.Gossip` for the Actor System's own gossip mechanism for membership dissemination public protocol GossipLogic { - associatedtype Gossip: GossipEnvelopeProtocol + associatedtype Gossip: Codable associatedtype Acknowledgement: Codable typealias Context = GossipLogicContext - // init(context: Context) // TODO: a form of context? - // ==== ------------------------------------------------------------------------------------------------------------ // MARK: Spreading gossip @@ -52,12 +50,28 @@ public protocol GossipLogic { /// /// Useful to implement using `PeerSelection` // TODO: OrderedSet would be the right thing here to be honest... - mutating func selectPeers(peers: [AddressableActorRef]) -> [AddressableActorRef] + mutating func selectPeers(_ peers: [AddressableActorRef]) -> [AddressableActorRef] // TODO: make a directive here - /// Allows for customizing the payload for specific targets + /// Allows for customizing the payload for each of the selected peers. + /// + /// Some gossip protocols are able to specialize the gossip payload sent to a specific peer, + /// e.g. by excluding information the peer is already aware of or similar. + /// + /// Returning `nil` means that the peer will be skipped in this gossip round, even though it was a candidate selected by peer selection. mutating func makePayload(target: AddressableActorRef) -> Gossip? + // ==== ------------------------------------------------------------------------------------------------------------ + // MARK: Receiving gossip + + /// Invoked whenever a gossip message is received from another peer. + /// + /// Note that a single gossiper instance may create _multiple_ `GossipLogic` instances, + /// one for each `GossipIdentifier` it is managing. This function is guaranteed to be invoked with the + /// gossip targeted to the same gossip identity as the logic's context + mutating func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? + + /// Invoked when the specific gossiped payload is acknowledged by the target. /// /// Note that acknowledgements may arrive in various orders, so make sure tracking them accounts for all possible orderings. @@ -70,10 +84,8 @@ public protocol GossipLogic { /// - gossip: mutating func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming gossip: Gossip) - // ==== ------------------------------------------------------------------------------------------------------------ - // MARK: Receiving gossip - - mutating func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? + // ==== ---------------------------------------------------------------------------------------------------------------- + // MARK: Local interactions / control messages mutating func receiveLocalGossipUpdate(_ gossip: Gossip) @@ -82,18 +94,26 @@ public protocol GossipLogic { } extension GossipLogic { + public mutating func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming gossip: Gossip) { + // ignore by default + } + public mutating func receiveSideChannelMessage(_ message: Any) throws { // ignore by default } } -public struct GossipLogicContext { +public struct GossipLogicContext { + /// Identifier associated with this gossip logic. + /// + /// Many gossipers only use a single identifier (and logic), + /// however some may need to manage gossip rounds for specific identifiers independently. public let gossipIdentifier: GossipIdentifier - private let ownerContext: ActorContext.Message> + private let gossiperContext: ActorContext.Message> internal init(ownerContext: ActorContext.Message>, gossipIdentifier: GossipIdentifier) { - self.ownerContext = ownerContext + self.gossiperContext = ownerContext self.gossipIdentifier = gossipIdentifier } @@ -102,21 +122,24 @@ public struct GossipLogicContext: GossipLogic, CustomStringConvertible { +public struct AnyGossipLogic: GossipLogic, CustomStringConvertible { @usableFromInline let _selectPeers: ([AddressableActorRef]) -> [AddressableActorRef] @usableFromInline @@ -131,10 +154,14 @@ public struct AnyGossipLogic Void + public init(context: Context) { + fatalError("\(Self.self) is intended to be created with a context, use `init(logic)` instead.") + } + public init(_ logic: Logic) where Logic: GossipLogic, Logic.Gossip == Gossip, Logic.Acknowledgement == Acknowledgement { var l = logic - self._selectPeers = { l.selectPeers(peers: $0) } + self._selectPeers = { l.selectPeers($0) } self._makePayload = { l.makePayload(target: $0) } self._receiveGossip = { l.receiveGossip($0, from: $1) } @@ -144,7 +171,7 @@ public struct AnyGossipLogic [AddressableActorRef] { + public func selectPeers(_ peers: [AddressableActorRef]) -> [AddressableActorRef] { self._selectPeers(peers) } @@ -156,8 +183,8 @@ public struct AnyGossipLogic` (inclusive) - public var gossipIntervalRandomFactor: Double = 0.2 { + public var intervalRandomFactor: Double = 0.2 { willSet { precondition(newValue >= 0, "settings.crdt.gossipIntervalRandomFactor MUST BE >= 0, was: \(newValue)") precondition(newValue <= 1, "settings.crdt.gossipIntervalRandomFactor MUST BE <= 1, was: \(newValue)") } } - public var effectiveGossipInterval: TimeAmount { - let baseInterval = self.gossipInterval - let randomizeMultiplier = Double.random(in: (1 - self.gossipIntervalRandomFactor) ... (1 + self.gossipIntervalRandomFactor)) + public var effectiveInterval: TimeAmount { + let baseInterval = self.interval + let randomizeMultiplier = Double.random(in: (1 - self.intervalRandomFactor) ... (1 + self.intervalRandomFactor)) let randomizedInterval = baseInterval * randomizeMultiplier return randomizedInterval } + /// Hints the Gossiper at weather or not acknowledgments are expected or not. + /// + /// If a gossiper which does not expect acknowledgements would be about to send an ack, a warning will be logged. + public var style: GossipSpreadingStyle + public enum GossipSpreadingStyle { + /// Gossip does NOT require acknowledgements and messages will be spread using uni-directional `tell` message sends. + case unidirectional + + /// Gossip DOES expect acknowledgements for spread messages, and messages will be spread using `ask` message sends. + case acknowledged(timeout: TimeAmount) + } + + + /// How the gossiper should discover peers to gossip with. public var peerDiscovery: PeerDiscovery = .manuallyIntroduced public enum PeerDiscovery { /// Peers have to be manually introduced by calling `control.introduce()` on to the gossiper. diff --git a/Sources/DistributedActors/Gossip/Gossip+Shell.swift b/Sources/DistributedActors/Gossip/Gossip+Shell.swift index 3d4c996e3..028d4fbcd 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Shell.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Shell.swift @@ -16,8 +16,10 @@ import Logging private let gossipTickKey: TimerKey = "gossip-tick" -/// Convergent gossip is a gossip mechanism which aims to equalize some state across all peers participating. -internal final class GossipShell { +/// :nodoc: +/// +/// Not intended to be spawned directly, use `Gossiper.spawn` instead! +internal final class GossipShell { typealias Ref = ActorRef let settings: Gossiper.Settings @@ -91,7 +93,7 @@ internal final class GossipShell, payload: Gossip, - ackRef: ActorRef + ackRef: ActorRef? ) { context.log.trace("Received gossip [\(identifier.gossipIdentifier)]", metadata: [ "gossip/identity": "\(identifier.gossipIdentifier)", @@ -101,8 +103,37 @@ internal final class GossipShell, Gossip, ackRef: ActorRef) + case gossip(identity: GossipIdentifier, origin: ActorRef, Gossip, ackRef: ActorRef?) // local messages case updatePayload(identifier: GossipIdentifier, Gossip) @@ -376,11 +411,10 @@ extension GossipShell { public enum Gossiper { /// Spawns a gossip actor, that will periodically gossip with its peers about the provided payload. static func start( - _ context: ActorRefFactory, name naming: ActorNaming, - of type: Envelope.Type = Envelope.self, - ofAcknowledgement acknowledgementType: Acknowledgement.Type = Acknowledgement.self, + _ context: ActorRefFactory, + name naming: ActorNaming, + settings: Settings, props: Props = .init(), - settings: Settings = .init(), makeLogic: @escaping (Logic.Context) -> Logic ) throws -> GossiperControl where Logic: GossipLogic, Logic.Gossip == Envelope, Logic.Acknowledgement == Acknowledgement { @@ -398,29 +432,34 @@ public enum Gossiper { // ==== ---------------------------------------------------------------------------------------------------------------- // MARK: GossiperControl -internal struct GossiperControl { - private let ref: GossipShell.Ref +/// Control object used to modify and interact with a spawned `Gossiper`. +public struct GossiperControl { + /// Internal FOR TESTING ONLY. + internal let ref: GossipShell.Ref - init(_ ref: GossipShell.Ref) { + init(_ ref: GossipShell.Ref) { self.ref = ref } - /// Introduce a peer to the gossip group - func introduce(peer: GossipShell.Ref) { + /// Introduce a peer to the gossip group. + /// + /// This method is fairly manual and error prone and as such internal only for the time being. + /// Please use the receptionist based peer discovery instead. + internal func introduce(peer: GossipShell.Ref) { self.ref.tell(.introducePeer(peer)) } // FIXME: is there some way to express that actually, Metadata is INSIDE Payload so I only want to pass the "envelope" myself...? - func update(_ identifier: GossipIdentifier, payload: GossipEnvelope) { + public func update(_ identifier: GossipIdentifier, payload: Gossip) { self.ref.tell(.updatePayload(identifier: identifier, payload)) } - func remove(_ identifier: GossipIdentifier) { + public func remove(_ identifier: GossipIdentifier) { self.ref.tell(.removePayload(identifier: identifier)) } /// Side channel messages which may be piped into specific gossip logics. - func sideChannelTell(_ identifier: GossipIdentifier, message: Any) { + public func sideChannelTell(_ identifier: GossipIdentifier, message: Any) { self.ref.tell(.sideChannelMessage(identifier: identifier, message)) } } diff --git a/Sources/DistributedActors/Gossip/PeerSelection.swift b/Sources/DistributedActors/Gossip/PeerSelection.swift index d40fc2847..7c2d1b981 100644 --- a/Sources/DistributedActors/Gossip/PeerSelection.swift +++ b/Sources/DistributedActors/Gossip/PeerSelection.swift @@ -27,37 +27,3 @@ public protocol PeerSelection { func select() -> Peers } - -// public struct StableRandomRoundRobin { -// -// var peerSet: Set -// var peers: [Peer] -// -// // how many peers we select in each gossip round, -// // we could for example be dynamic and notice if we have 10+ nodes, we pick 2 members to speed up the dissemination etc. -// let n = 1 -// -// public init() { -// } -// -// func onMembershipEvent(event: Cluster.Event) { -// -// } -// -// func update(peers newPeers: [Peer]) { -// let newPeerSet = Set(peers) -// } -// -// func select() -> [Peer] { -// var selectedPeers: [AddressableActorRef] = [] -// selectedPeers.reserveCapacity(n) -// -// for peer in peers.shuffled() -// where selectedPeers.count < n && self.shouldGossipWith(peer) { -// selectedPeers.append(peer) -// } -// -// return selectedPeers -// } -// -// } diff --git a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift index 6a706a8a2..6010bd89d 100644 --- a/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift +++ b/Tests/DistributedActorsTests/Cluster/MembershipGossipLogicSimulationTests.swift @@ -275,7 +275,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas }, stopRunWhen: { logics, _ in logics.allSatisfy { logic in - logic.selectPeers(peers: self.peers(of: logic)) == [] // no more peers to talk to + logic.selectPeers(self.peers(of: logic)) == [] // no more peers to talk to } }, assert: { results in @@ -347,7 +347,7 @@ final class MembershipGossipLogicSimulationTests: ClusteredActorSystemsXCTestCas // information. let participatingGossips = self.logics.shuffled() for logic in participatingGossips { - let selectedPeers: [AddressableActorRef] = logic.selectPeers(peers: self.peers(of: logic)) + let selectedPeers: [AddressableActorRef] = logic.selectPeers(self.peers(of: logic)) log.notice("[\(logic.nodeName)] selected peers: \(selectedPeers.map { $0.address.node!.node.systemName })") for targetPeer in selectedPeers { diff --git a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift b/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift index 0a3ec1288..764e9f805 100644 --- a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift +++ b/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift @@ -19,21 +19,32 @@ import NIOSSL import XCTest final class GossipShellTests: ActorSystemXCTestCase { + + func peerBehavior() -> Behavior.Message> { + .receiveMessage { msg in + if "\(msg)".contains("stop") { return .stop } else { return .same } + } + } + + // ==== ---------------------------------------------------------------------------------------------------------------- + // MARK: test_down_beGossipedToOtherNodes + func test_down_beGossipedToOtherNodes() throws { let p = self.testKit.spawnTestProbe(expecting: [AddressableActorRef].self) let control = try Gossiper.start( self.system, name: "gossiper", - settings: .init(gossipInterval: .seconds(1)), - makeLogic: { _ in InspectOfferedPeersTestGossipLogic(offeredPeersProbe: p.ref) } - ) + settings: .init( + interval: .seconds(1), + style: .unidirectional + )) { _ in InspectOfferedPeersTestGossipLogic(offeredPeersProbe: p.ref) } - let peerBehavior: Behavior.Message> = .receiveMessage { msg in - if "\(msg)".contains("stop") { return .stop } else { return .same } - } - let first = try self.system.spawn("first", peerBehavior) - let second = try self.system.spawn("second", peerBehavior) + + let first: ActorRef.Message> = + try self.system.spawn("first", self.peerBehavior()) + let second: ActorRef.Message> = + try self.system.spawn("second", self.peerBehavior()) control.introduce(peer: first) control.introduce(peer: second) @@ -49,7 +60,7 @@ final class GossipShellTests: ActorSystemXCTestCase { } struct InspectOfferedPeersTestGossipLogic: GossipLogic { - struct Gossip: GossipEnvelopeProtocol { + struct Gossip: Codable { let metadata: String let payload: String @@ -66,7 +77,7 @@ final class GossipShellTests: ActorSystemXCTestCase { self.offeredPeersProbe = offeredPeersProbe } - func selectPeers(peers: [AddressableActorRef]) -> [AddressableActorRef] { + func selectPeers(_ peers: [AddressableActorRef]) -> [AddressableActorRef] { self.offeredPeersProbe.tell(peers) return [] } @@ -75,7 +86,75 @@ final class GossipShellTests: ActorSystemXCTestCase { nil } - func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming envelope: Gossip) {} + func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming gossip: Gossip) {} + + func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? { + nil + } + + func receiveLocalGossipUpdate(_ gossip: Gossip) {} + } + + // ==== ---------------------------------------------------------------------------------------------------------------- + // MARK: test_unidirectional_yetEmitsAck_shouldWarn + + func test_unidirectional_yetEmitsAck_shouldWarn() throws { + let p = self.testKit.spawnTestProbe(expecting: String.self) + + let control = try Gossiper.start( + self.system, + name: "noAcks", + settings: .init( + interval: .milliseconds(100), + style: .unidirectional + ), + makeLogic: { _ in NoAcksTestGossipLogic(probe: p.ref) } + ) + + let first: ActorRef.Message> = + try self.system.spawn("first", self.peerBehavior()) + + control.introduce(peer: first) + control.update(StringGossipIdentifier("hi"), payload: .init("hello")) + control.ref.tell( + .gossip( + identity: StringGossipIdentifier("example"), + origin: first, .init("unexpected"), + ackRef: system.deadLetters.adapted() // this is wrong on purpose; we're configured as `unidirectional`; this should cause warnings + ) + ) + + try self.logCapture.awaitLogContaining(self.testKit, + text: " Incoming gossip has acknowledgement actor ref and seems to be expecting an ACK, while this gossiper is configured as .unidirectional!" + ) + } + + struct NoAcksTestGossipLogic: GossipLogic { + struct Gossip: Codable { + let metadata: String + let payload: String + + init(_ info: String) { + self.metadata = info + self.payload = info + } + } + + let probe: ActorRef + + typealias Acknowledgement = String + + func selectPeers(_ peers: [AddressableActorRef]) -> [AddressableActorRef] { + peers + } + + func makePayload(target: AddressableActorRef) -> Gossip? { + .init("Hello") // legal but will produce a warning + } + + func receiveAcknowledgement(_ acknowledgement: Acknowledgement, from peer: AddressableActorRef, confirming gossip: Gossip) { + self.probe.tell("un-expected acknowledgement: \(acknowledgement) from \(peer) confirming \(gossip)") + } func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? { nil From a146889309ad0622413a991562939d6ca0089040 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Wed, 24 Jun 2020 15:11:51 +0900 Subject: [PATCH 12/15] =gossiper renames, cleanup --- .../CRDT/CRDT+ReplicatorShell.swift | 2 +- .../Cluster/ClusterShell.swift | 2 +- .../Cluster+MembershipGossip.swift | 1 - .../Gossip/Gossip+Settings.swift | 1 - .../{Gossip+Logic.swift => GossipLogic.swift} | 1 - ...ift => Gossiper+Shell+Serialization.swift} | 0 ...ossip+Shell.swift => Gossiper+Shell.swift} | 146 ++---------------- .../DistributedActors/Gossip/Gossiper.swift | 146 ++++++++++++++++++ ...llTests.swift => GossiperShellTests.swift} | 18 +-- 9 files changed, 168 insertions(+), 149 deletions(-) rename Sources/DistributedActors/Gossip/{Gossip+Logic.swift => GossipLogic.swift} (99%) rename Sources/DistributedActors/Gossip/{Gossip+Shell+Coding.swift => Gossiper+Shell+Serialization.swift} (100%) rename Sources/DistributedActors/Gossip/{Gossip+Shell.swift => Gossiper+Shell.swift} (76%) create mode 100644 Sources/DistributedActors/Gossip/Gossiper.swift rename Tests/DistributedActorsTests/Gossip/{GossipShellTests.swift => GossiperShellTests.swift} (93%) diff --git a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift index 3d326b825..6b7171f2a 100644 --- a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift +++ b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift @@ -71,7 +71,7 @@ extension CRDT.Replicator { } ) - self.gossipReplication = try Gossiper.start( + self.gossipReplication = try Gossiper.spawn( context, name: "gossip", settings: Gossiper.Settings( diff --git a/Sources/DistributedActors/Cluster/ClusterShell.swift b/Sources/DistributedActors/Cluster/ClusterShell.swift index 1962dca2b..7a60db481 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell.swift @@ -422,7 +422,7 @@ extension ClusterShell { return context.awaitResultThrowing(of: chanElf, timeout: clusterSettings.bindTimeout) { (chan: Channel) in context.log.info("Bound to \(chan.localAddress.map { $0.description } ?? "")") - let gossiperControl: GossiperControl = try Gossiper.start( + let gossiperControl: GossiperControl = try Gossiper.spawn( context, name: "\(ActorPath._clusterGossip.name)", settings: .init( diff --git a/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift index a0c061cd4..c72b9a904 100644 --- a/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift +++ b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift @@ -157,7 +157,6 @@ extension Cluster { return !laggingBehindMemberFound } } - } extension Cluster.MembershipGossip: CustomPrettyStringConvertible {} diff --git a/Sources/DistributedActors/Gossip/Gossip+Settings.swift b/Sources/DistributedActors/Gossip/Gossip+Settings.swift index 0db33b6b5..823b46964 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Settings.swift +++ b/Sources/DistributedActors/Gossip/Gossip+Settings.swift @@ -53,7 +53,6 @@ extension Gossiper { case acknowledged(timeout: TimeAmount) } - /// How the gossiper should discover peers to gossip with. public var peerDiscovery: PeerDiscovery = .manuallyIntroduced public enum PeerDiscovery { diff --git a/Sources/DistributedActors/Gossip/Gossip+Logic.swift b/Sources/DistributedActors/Gossip/GossipLogic.swift similarity index 99% rename from Sources/DistributedActors/Gossip/Gossip+Logic.swift rename to Sources/DistributedActors/Gossip/GossipLogic.swift index 3820acd7e..6c2b507a2 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Logic.swift +++ b/Sources/DistributedActors/Gossip/GossipLogic.swift @@ -71,7 +71,6 @@ public protocol GossipLogic { /// gossip targeted to the same gossip identity as the logic's context mutating func receiveGossip(_ gossip: Gossip, from peer: AddressableActorRef) -> Acknowledgement? - /// Invoked when the specific gossiped payload is acknowledged by the target. /// /// Note that acknowledgements may arrive in various orders, so make sure tracking them accounts for all possible orderings. diff --git a/Sources/DistributedActors/Gossip/Gossip+Shell+Coding.swift b/Sources/DistributedActors/Gossip/Gossiper+Shell+Serialization.swift similarity index 100% rename from Sources/DistributedActors/Gossip/Gossip+Shell+Coding.swift rename to Sources/DistributedActors/Gossip/Gossiper+Shell+Serialization.swift diff --git a/Sources/DistributedActors/Gossip/Gossip+Shell.swift b/Sources/DistributedActors/Gossip/Gossiper+Shell.swift similarity index 76% rename from Sources/DistributedActors/Gossip/Gossip+Shell.swift rename to Sources/DistributedActors/Gossip/Gossiper+Shell.swift index 028d4fbcd..a0a7fc140 100644 --- a/Sources/DistributedActors/Gossip/Gossip+Shell.swift +++ b/Sources/DistributedActors/Gossip/Gossiper+Shell.swift @@ -32,7 +32,7 @@ internal final class GossipShell { typealias PeerRef = ActorRef private var peers: Set - fileprivate init( + internal init( settings: Gossiper.Settings, makeLogic: @escaping (Logic.Context) -> Logic ) where Logic: GossipLogic, Logic.Gossip == Gossip, Logic.Acknowledgement == Acknowledgement { @@ -118,10 +118,11 @@ internal final class GossipShell { GossipLogic attempted to offer Acknowledgement while it is configured as .unidirectional!\ This is potentially a bug in the logic or the Gossiper's configuration. Dropping acknowledgement. """, metadata: [ - "gossip/identity": "\(identifier.gossipIdentifier)", - "gossip/origin": "\(origin.address)", - "gossip/ack": "\(unexpectedAck)", - ]) + "gossip/identity": "\(identifier.gossipIdentifier)", + "gossip/origin": "\(origin.address)", + "gossip/ack": "\(unexpectedAck)", + ] + ) } if let unexpectedAckRef = ackRef { context.log.warning( @@ -129,10 +130,11 @@ internal final class GossipShell { Incoming gossip has acknowledgement actor ref and seems to be expecting an ACK, while this gossiper is configured as .unidirectional! \ This is potentially a bug in the logic or the Gossiper's configuration. """, metadata: [ - "gossip/identity": "\(identifier.gossipIdentifier)", - "gossip/origin": "\(origin.address)", - "gossip/ackRef": "\(unexpectedAckRef)", - ]) + "gossip/identity": "\(identifier.gossipIdentifier)", + "gossip/origin": "\(origin.address)", + "gossip/ackRef": "\(unexpectedAckRef)", + ] + ) } } } @@ -403,129 +405,3 @@ extension GossipShell { case _periodicGossipTick } } - -// ==== ---------------------------------------------------------------------------------------------------------------- -// MARK: Gossiper - -/// A Gossiper -public enum Gossiper { - /// Spawns a gossip actor, that will periodically gossip with its peers about the provided payload. - static func start( - _ context: ActorRefFactory, - name naming: ActorNaming, - settings: Settings, - props: Props = .init(), - makeLogic: @escaping (Logic.Context) -> Logic - ) throws -> GossiperControl - where Logic: GossipLogic, Logic.Gossip == Envelope, Logic.Acknowledgement == Acknowledgement { - let ref = try context.spawn( - naming, - of: GossipShell.Message.self, - props: props, - file: #file, line: #line, - GossipShell(settings: settings, makeLogic: makeLogic).behavior - ) - return GossiperControl(ref) - } -} - -// ==== ---------------------------------------------------------------------------------------------------------------- -// MARK: GossiperControl - -/// Control object used to modify and interact with a spawned `Gossiper`. -public struct GossiperControl { - /// Internal FOR TESTING ONLY. - internal let ref: GossipShell.Ref - - init(_ ref: GossipShell.Ref) { - self.ref = ref - } - - /// Introduce a peer to the gossip group. - /// - /// This method is fairly manual and error prone and as such internal only for the time being. - /// Please use the receptionist based peer discovery instead. - internal func introduce(peer: GossipShell.Ref) { - self.ref.tell(.introducePeer(peer)) - } - - // FIXME: is there some way to express that actually, Metadata is INSIDE Payload so I only want to pass the "envelope" myself...? - public func update(_ identifier: GossipIdentifier, payload: Gossip) { - self.ref.tell(.updatePayload(identifier: identifier, payload)) - } - - public func remove(_ identifier: GossipIdentifier) { - self.ref.tell(.removePayload(identifier: identifier)) - } - - /// Side channel messages which may be piped into specific gossip logics. - public func sideChannelTell(_ identifier: GossipIdentifier, message: Any) { - self.ref.tell(.sideChannelMessage(identifier: identifier, message)) - } -} - -// ==== ---------------------------------------------------------------------------------------------------------------- -// MARK: Gossip Identifier - -/// Used to identify which identity a payload is tied with. -/// E.g. it could be used to mark the CRDT instance the gossip is carrying, or which "entity" a gossip relates to. -// FIXME: just force GossipIdentifier to be codable, avoid this hacky dance? -public protocol GossipIdentifier { - var gossipIdentifier: String { get } - - init(_ gossipIdentifier: String) - - var asAnyGossipIdentifier: AnyGossipIdentifier { get } -} - -public struct AnyGossipIdentifier: Hashable, GossipIdentifier { - public let underlying: GossipIdentifier - - public init(_ id: String) { - self.underlying = StringGossipIdentifier(stringLiteral: id) - } - - public init(_ identifier: GossipIdentifier) { - if let any = identifier as? AnyGossipIdentifier { - self = any - } else { - self.underlying = identifier - } - } - - public var gossipIdentifier: String { - self.underlying.gossipIdentifier - } - - public var asAnyGossipIdentifier: AnyGossipIdentifier { - self - } - - public func hash(into hasher: inout Hasher) { - self.underlying.gossipIdentifier.hash(into: &hasher) - } - - public static func == (lhs: AnyGossipIdentifier, rhs: AnyGossipIdentifier) -> Bool { - lhs.underlying.gossipIdentifier == rhs.underlying.gossipIdentifier - } -} - -public struct StringGossipIdentifier: GossipIdentifier, Hashable, ExpressibleByStringLiteral, CustomStringConvertible { - public let gossipIdentifier: String - - public init(_ gossipIdentifier: StringLiteralType) { - self.gossipIdentifier = gossipIdentifier - } - - public init(stringLiteral gossipIdentifier: StringLiteralType) { - self.gossipIdentifier = gossipIdentifier - } - - public var asAnyGossipIdentifier: AnyGossipIdentifier { - AnyGossipIdentifier(self) - } - - public var description: String { - "StringGossipIdentifier(\(self.gossipIdentifier))" - } -} diff --git a/Sources/DistributedActors/Gossip/Gossiper.swift b/Sources/DistributedActors/Gossip/Gossiper.swift new file mode 100644 index 000000000..88b52b83f --- /dev/null +++ b/Sources/DistributedActors/Gossip/Gossiper.swift @@ -0,0 +1,146 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Distributed Actors open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift Distributed Actors project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.md for the list of Swift Distributed Actors project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Logging + +// ==== ---------------------------------------------------------------------------------------------------------------- +// MARK: Gossiper + +/// A generalized Gossiper which can interpret a `GossipLogic` provided to it. +/// +/// It encapsulates multiple error prone details surrounding implementing gossip mechanisms, +/// such as peer monitoring and managing cluster events and their impact on peers. +/// +/// It can automatically discover new peers as new members join the cluster using the `Receptionist`. +public enum Gossiper { + /// Spawns a gossip actor, that will periodically gossip with its peers about the provided payload. + static func spawn( + _ context: ActorRefFactory, + name naming: ActorNaming, + settings: Settings, + props: Props = .init(), + makeLogic: @escaping (Logic.Context) -> Logic + ) throws -> GossiperControl + where Logic: GossipLogic, Logic.Gossip == Envelope, Logic.Acknowledgement == Acknowledgement { + let ref = try context.spawn( + naming, + of: GossipShell.Message.self, + props: props, + file: #file, line: #line, + GossipShell(settings: settings, makeLogic: makeLogic).behavior + ) + return GossiperControl(ref) + } +} + +// ==== ---------------------------------------------------------------------------------------------------------------- +// MARK: GossiperControl + +/// Control object used to modify and interact with a spawned `Gossiper`. +public struct GossiperControl { + /// Internal FOR TESTING ONLY. + internal let ref: GossipShell.Ref + + init(_ ref: GossipShell.Ref) { + self.ref = ref + } + + /// Introduce a peer to the gossip group. + /// + /// This method is fairly manual and error prone and as such internal only for the time being. + /// Please use the receptionist based peer discovery instead. + internal func introduce(peer: GossipShell.Ref) { + self.ref.tell(.introducePeer(peer)) + } + + // FIXME: is there some way to express that actually, Metadata is INSIDE Payload so I only want to pass the "envelope" myself...? + public func update(_ identifier: GossipIdentifier, payload: Gossip) { + self.ref.tell(.updatePayload(identifier: identifier, payload)) + } + + public func remove(_ identifier: GossipIdentifier) { + self.ref.tell(.removePayload(identifier: identifier)) + } + + /// Side channel messages which may be piped into specific gossip logics. + public func sideChannelTell(_ identifier: GossipIdentifier, message: Any) { + self.ref.tell(.sideChannelMessage(identifier: identifier, message)) + } +} + +// ==== ---------------------------------------------------------------------------------------------------------------- +// MARK: Gossip Identifier + +/// Used to identify which identity a payload is tied with. +/// E.g. it could be used to mark the CRDT instance the gossip is carrying, or which "entity" a gossip relates to. +// FIXME: just force GossipIdentifier to be codable, avoid this hacky dance? +public protocol GossipIdentifier { + var gossipIdentifier: String { get } + + init(_ gossipIdentifier: String) + + var asAnyGossipIdentifier: AnyGossipIdentifier { get } +} + +public struct AnyGossipIdentifier: Hashable, GossipIdentifier { + public let underlying: GossipIdentifier + + public init(_ id: String) { + self.underlying = StringGossipIdentifier(stringLiteral: id) + } + + public init(_ identifier: GossipIdentifier) { + if let any = identifier as? AnyGossipIdentifier { + self = any + } else { + self.underlying = identifier + } + } + + public var gossipIdentifier: String { + self.underlying.gossipIdentifier + } + + public var asAnyGossipIdentifier: AnyGossipIdentifier { + self + } + + public func hash(into hasher: inout Hasher) { + self.underlying.gossipIdentifier.hash(into: &hasher) + } + + public static func == (lhs: AnyGossipIdentifier, rhs: AnyGossipIdentifier) -> Bool { + lhs.underlying.gossipIdentifier == rhs.underlying.gossipIdentifier + } +} + +public struct StringGossipIdentifier: GossipIdentifier, Hashable, ExpressibleByStringLiteral, CustomStringConvertible { + public let gossipIdentifier: String + + public init(_ gossipIdentifier: StringLiteralType) { + self.gossipIdentifier = gossipIdentifier + } + + public init(stringLiteral gossipIdentifier: StringLiteralType) { + self.gossipIdentifier = gossipIdentifier + } + + public var asAnyGossipIdentifier: AnyGossipIdentifier { + AnyGossipIdentifier(self) + } + + public var description: String { + "StringGossipIdentifier(\(self.gossipIdentifier))" + } +} diff --git a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift b/Tests/DistributedActorsTests/Gossip/GossiperShellTests.swift similarity index 93% rename from Tests/DistributedActorsTests/Gossip/GossipShellTests.swift rename to Tests/DistributedActorsTests/Gossip/GossiperShellTests.swift index 764e9f805..dc30c1477 100644 --- a/Tests/DistributedActorsTests/Gossip/GossipShellTests.swift +++ b/Tests/DistributedActorsTests/Gossip/GossiperShellTests.swift @@ -18,10 +18,9 @@ import Foundation import NIOSSL import XCTest -final class GossipShellTests: ActorSystemXCTestCase { - +final class GossiperShellTests: ActorSystemXCTestCase { func peerBehavior() -> Behavior.Message> { - .receiveMessage { msg in + .receiveMessage { msg in if "\(msg)".contains("stop") { return .stop } else { return .same } } } @@ -32,14 +31,14 @@ final class GossipShellTests: ActorSystemXCTestCase { func test_down_beGossipedToOtherNodes() throws { let p = self.testKit.spawnTestProbe(expecting: [AddressableActorRef].self) - let control = try Gossiper.start( + let control = try Gossiper.spawn( self.system, name: "gossiper", settings: .init( interval: .seconds(1), style: .unidirectional - )) { _ in InspectOfferedPeersTestGossipLogic(offeredPeersProbe: p.ref) } - + ) + ) { _ in InspectOfferedPeersTestGossipLogic(offeredPeersProbe: p.ref) } let first: ActorRef.Message> = try self.system.spawn("first", self.peerBehavior()) @@ -98,10 +97,10 @@ final class GossipShellTests: ActorSystemXCTestCase { // ==== ---------------------------------------------------------------------------------------------------------------- // MARK: test_unidirectional_yetEmitsAck_shouldWarn - func test_unidirectional_yetEmitsAck_shouldWarn() throws { + func test_unidirectional_yetReceivesAckRef_shouldWarn() throws { let p = self.testKit.spawnTestProbe(expecting: String.self) - let control = try Gossiper.start( + let control = try Gossiper.spawn( self.system, name: "noAcks", settings: .init( @@ -124,7 +123,8 @@ final class GossipShellTests: ActorSystemXCTestCase { ) ) - try self.logCapture.awaitLogContaining(self.testKit, + try self.logCapture.awaitLogContaining( + self.testKit, text: " Incoming gossip has acknowledgement actor ref and seems to be expecting an ACK, while this gossiper is configured as .unidirectional!" ) } From 36011709ba937c67cefa868f94ca5df47717d09f Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Wed, 24 Jun 2020 15:20:15 +0900 Subject: [PATCH 13/15] =crdt,gossip simplify gossip type for CRDTs, it has all things we need --- .../DistributedActors/CRDT/CRDT+Gossip.swift | 21 +++---------------- .../CRDT/CRDT+ReplicatorShell.swift | 11 +++------- .../Serialization/Serialization.swift | 1 - 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift index 0a6c4ab6b..95215d544 100644 --- a/Sources/DistributedActors/CRDT/CRDT+Gossip.swift +++ b/Sources/DistributedActors/CRDT/CRDT+Gossip.swift @@ -181,14 +181,9 @@ extension CRDT.Identity: GossipIdentifier { extension CRDT { /// The gossip to be spread about a specific CRDT (identity). struct Gossip: Codable, CustomStringConvertible, CustomPrettyStringConvertible { - struct Metadata: Codable {} // FIXME: remove, seems we dont need metadata explicitly here - typealias Payload = StateBasedCRDT + var payload: StateBasedCRDT - var metadata: Metadata - var payload: Payload - - init(metadata: CRDT.Gossip.Metadata, payload: CRDT.Gossip.Payload) { - self.metadata = metadata + init(payload: StateBasedCRDT) { self.payload = payload } @@ -201,7 +196,7 @@ extension CRDT { } var description: String { - "CRDT.Gossip(metadata: \(metadata), payload: \(payload))" + "CRDT.Gossip(\(payload))" } } @@ -210,8 +205,6 @@ extension CRDT { extension CRDT.Gossip { enum CodingKeys: CodingKey { - case metadata - case metadataManifest case payload case payloadManifest } @@ -223,10 +216,6 @@ extension CRDT.Gossip { let container = try decoder.container(keyedBy: CodingKeys.self) - let manifestData = try container.decode(Data.self, forKey: .metadata) - let manifestManifest = try container.decode(Serialization.Manifest.self, forKey: .metadataManifest) - self.metadata = try context.serialization.deserialize(as: Metadata.self, from: .data(manifestData), using: manifestManifest) - let payloadData = try container.decode(Data.self, forKey: .payload) let payloadManifest = try container.decode(Serialization.Manifest.self, forKey: .payloadManifest) self.payload = try context.serialization.deserialize(as: StateBasedCRDT.self, from: .data(payloadData), using: payloadManifest) @@ -242,9 +231,5 @@ extension CRDT.Gossip { let serializedPayload = try context.serialization.serialize(self.payload) try container.encode(serializedPayload.buffer.readData(), forKey: .payload) try container.encode(serializedPayload.manifest, forKey: .payloadManifest) - - let serializedMetadata = try context.serialization.serialize(self.metadata) - try container.encode(serializedMetadata.buffer.readData(), forKey: .metadata) - try container.encode(serializedMetadata.manifest, forKey: .metadataManifest) } } diff --git a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift index 6b7171f2a..622ca3f67 100644 --- a/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift +++ b/Sources/DistributedActors/CRDT/CRDT+ReplicatorShell.swift @@ -268,10 +268,8 @@ extension CRDT.Replicator { _ id: CRDT.Identity, _ data: StateBasedCRDT ) { - let gossip = CRDT.Gossip( - metadata: .init(), - payload: data // TODO: v2, allow tracking the deltas here - ) + // TODO: v2, allow tracking the deltas here + let gossip = CRDT.Gossip(payload: data) self.gossipReplication.update(id, payload: gossip) } @@ -408,10 +406,7 @@ extension CRDT.Replicator { replyTo.tell(.success(updatedData)) // Update the data stored in the replicator (yeah today we store 2 copies in the replicators, we could converge them into one with enough effort) - let gossip = CRDT.Gossip( - metadata: .init(), - payload: updatedData - ) + let gossip = CRDT.Gossip(payload: updatedData) self.gossipReplication.update(id, payload: gossip) // TODO: v2, allow tracking the deltas here // Followed by notifying all owners since the CRDT might have been updated diff --git a/Sources/DistributedActors/Serialization/Serialization.swift b/Sources/DistributedActors/Serialization/Serialization.swift index 08f6ccc47..05e869bc4 100644 --- a/Sources/DistributedActors/Serialization/Serialization.swift +++ b/Sources/DistributedActors/Serialization/Serialization.swift @@ -169,7 +169,6 @@ public class Serialization { settings.register(CRDT.GossipAck.self) settings.register(GossipShell.Message.self) settings.register(CRDT.Gossip.self) - settings.register(CRDT.Gossip.Metadata.self) // errors settings.register(ErrorEnvelope.self) // TODO: can be removed once https://github.com/apple/swift/pull/30318 lands From 7533557f48bc0c35879215521ac56849f819bf6c Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Wed, 24 Jun 2020 16:10:20 +0900 Subject: [PATCH 14/15] =cluster,gossip,leader Stronger guarantees on first leader then up-moves ordering observed on event stream --- .../Cluster/ClusterShell+LeaderActions.swift | 5 ++- .../Cluster/ClusterShell.swift | 33 +++++++++++-------- .../DistributedActors/Gossip/Gossiper.swift | 4 +++ .../ActorTestKit.swift | 6 +++- .../Cluster/AssociationClusteredTests.swift | 7 ++-- 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift b/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift index ec9e0666a..1aaa67e10 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift @@ -65,7 +65,8 @@ extension ClusterShellState { extension ClusterShell { func interpretLeaderActions( - _ system: ActorSystem, _ previousState: ClusterShellState, + _ system: ActorSystem, + _ previousState: ClusterShellState, _ leaderActions: [ClusterShellState.LeaderAction], file: String = #file, line: UInt = #line ) -> ClusterShellState { @@ -118,6 +119,8 @@ extension ClusterShell { ] ) + system.cluster.updateMembershipSnapshot(state.membership) + return state } diff --git a/Sources/DistributedActors/Cluster/ClusterShell.swift b/Sources/DistributedActors/Cluster/ClusterShell.swift index 7a60db481..f0874c44c 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell.swift @@ -550,7 +550,23 @@ extension ClusterShell { self.tracelog(context, .receive(from: state.localNode.node), message: event) var state = state - let changeDirective = state.applyClusterEvent(event) + // 1) IMPORTANT: We MUST apply and act on the incoming event FIRST, before handling the other events. + // This is because: + // - is the event is a `leadershipChange` applying it could make us become the leader + // - if that happens, the 2) phase will collect leader actions (perhaps for the first time), + // and issue any pending up/down events. + // - For consistency such events MUST only be issued AFTER we have emitted the leadership change (!) + // Otherwise subscribers may end up seeing joining->up changes BEFORE they see the leadershipChange, + // which is not strictly wrong per se, however it is very confusing -- we know there MUST be a leader + // somewhere in order to perform those moves, so it is confusing if such joining->up events were to be + // seen by a subscriber before they saw "we have a leader". + if state.applyClusterEvent(event).applied { + state.latestGossip.incrementOwnerVersion() + // we only publish the event if it really caused a change in membership, to avoid echoing "the same" change many times. + self.clusterEvents.publish(event) + } // else no "effective change", thus we do not publish events + + // 2) Collect and interpret leader actions which may result changing the membership and publishing events for the changes let actions: [ClusterShellState.LeaderAction] = state.collectLeaderActions() state = self.interpretLeaderActions(context.system, state, actions) @@ -558,14 +574,6 @@ extension ClusterShell { self.tryIntroduceGossipPeer(context, state, change: change) } - if changeDirective.applied { - state.latestGossip.incrementOwnerVersion() - // update the membership snapshot before publishing change events - context.system.cluster.updateMembershipSnapshot(state.membership) - // we only publish the event if it really caused a change in membership, to avoid echoing "the same" change many times. - self.clusterEvents.publish(event) - } // else no "effective change", thus we do not publish events - return self.ready(state: state) } @@ -604,11 +612,9 @@ extension ClusterShell { } let leaderActions = state.collectLeaderActions() - if !leaderActions.isEmpty { - state.log.trace("Leadership actions upon gossip: \(leaderActions)", metadata: ["tag": "membership"]) - } - state = self.interpretLeaderActions(context.system, state, leaderActions) + + // definitely update the snapshot; even if no leader actions performed context.system.cluster.updateMembershipSnapshot(state.membership) return self.ready(state: state) @@ -1240,7 +1246,6 @@ extension ClusterShell { } state = self.interpretLeaderActions(context.system, state, state.collectLeaderActions()) - return state } diff --git a/Sources/DistributedActors/Gossip/Gossiper.swift b/Sources/DistributedActors/Gossip/Gossiper.swift index 88b52b83f..cabc68525 100644 --- a/Sources/DistributedActors/Gossip/Gossiper.swift +++ b/Sources/DistributedActors/Gossip/Gossiper.swift @@ -23,6 +23,10 @@ import Logging /// such as peer monitoring and managing cluster events and their impact on peers. /// /// It can automatically discover new peers as new members join the cluster using the `Receptionist`. +/// +/// - SeeAlso: [Gossiping in Distributed Systems](https://www.distributed-systems.net/my-data/papers/2007.osr.pdf) (Anne-Marie Kermarrec, Maarten van Steen), +/// for a nice overview of the general concepts involved in gossip algorithms. +/// - SeeAlso: [Cassandra Internals — Understanding Gossip](https://www.youtube.com/watch?v=FuP1Fvrv6ZQ) which a nice generally useful talk public enum Gossiper { /// Spawns a gossip actor, that will periodically gossip with its peers about the provided payload. static func spawn( diff --git a/Sources/DistributedActorsTestKit/ActorTestKit.swift b/Sources/DistributedActorsTestKit/ActorTestKit.swift index 84ef86a9c..68cff3f90 100644 --- a/Sources/DistributedActorsTestKit/ActorTestKit.swift +++ b/Sources/DistributedActorsTestKit/ActorTestKit.swift @@ -237,7 +237,11 @@ public struct EventuallyError: Error, CustomStringConvertible, CustomDebugString } public var debugDescription: String { - "EventuallyError(callSite: \(self.callSite), timeAmount: \(self.timeAmount), polledTimes: \(self.polledTimes), lastError: \(optional: self.lastError))" + let error = self.callSite.error( + """ + Eventually block failed, after \(self.timeAmount) (polled \(self.polledTimes) times), last error: \(optional: self.lastError) + """) + return "\(error)" } } diff --git a/Tests/DistributedActorsTests/Cluster/AssociationClusteredTests.swift b/Tests/DistributedActorsTests/Cluster/AssociationClusteredTests.swift index 6dee70af3..55fdf3464 100644 --- a/Tests/DistributedActorsTests/Cluster/AssociationClusteredTests.swift +++ b/Tests/DistributedActorsTests/Cluster/AssociationClusteredTests.swift @@ -158,10 +158,11 @@ final class ClusterAssociationTests: ClusteredActorSystemsXCTestCase { alone.cluster.join(node: alone.cluster.node.node) // "self join", should simply be ignored let testKit = self.testKit(alone) - - sleep(1) try testKit.eventually(within: .seconds(3)) { - alone.cluster.membershipSnapshot.count.shouldEqual(1) + let snapshot: Cluster.Membership = alone.cluster.membershipSnapshot + if snapshot.count != 1 { + throw TestError("Expected membership to include self node, was: \(snapshot)") + } } } From 4893426510a7a32926a07cd99156a1d76232f432 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Wed, 24 Jun 2020 20:15:08 +0900 Subject: [PATCH 15/15] =leaderactions stricter convergence requirement --- .../Cluster/ClusterShell+LeaderActions.swift | 15 +-------------- .../Cluster+MembershipGossip.swift | 2 +- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift b/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift index 1aaa67e10..3e8164773 100644 --- a/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift +++ b/Sources/DistributedActors/Cluster/ClusterShell+LeaderActions.swift @@ -27,7 +27,7 @@ extension ClusterShellState { } guard self.latestGossip.converged() else { - return [] // leader actions are only performed when + return [] // leader actions are only performed when up nodes are converged } func collectMemberUpMoves() -> [LeaderAction] { @@ -74,19 +74,6 @@ extension ClusterShell { return previousState } - guard previousState.latestGossip.converged() else { - previousState.log.warning( - "Skipping leader actions, gossip not converged", - metadata: [ - "tag": "leader-action", - "leader/actions": "\(leaderActions)", - "gossip/current": "\(previousState.latestGossip)", - "leader/interpret/location": "\(file):\(line)", - ] - ) - return previousState - } - var state = previousState state.log.trace( "Performing leader actions: \(leaderActions)", diff --git a/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift index c72b9a904..9bf08555e 100644 --- a/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift +++ b/Sources/DistributedActors/Cluster/MembershipGossip/Cluster+MembershipGossip.swift @@ -134,7 +134,7 @@ extension Cluster { /// Only `.up` and `.leaving` members are considered, since joining members are "too early" /// to matter in decisions, and down members shall never participate in decision making. func converged() -> Bool { - let members = self.membership.members(withStatus: [.up, .leaving]) // FIXME: we should not require joining nodes in convergence, can losen up a bit here I hope + let members = self.membership.members(withStatus: [.joining, .up, .leaving]) // FIXME: we should not require joining nodes in convergence, can losen up a bit here I hope let requiredVersion = self.version if members.isEmpty {