async demotion failure in sim-testnet

[coot]

    async demotion:                                             FAIL (17.84s)
      *** Failed! Falsified (after 48 tests and 8 shrinks):
      AbsBearerInfo {abiConnectionDelay = SmallDelay, abiInboundAttenuation = NoAttenuation FastSpeed, abiOutboundAttenuation = NoAttenuation FastSpeed, abiInboundWriteFailure = Just 8, abiOutboundWriteFailure = Nothing, abiAcceptFailure = Nothing, abiSDUSize = LargeSDU}
      DiffusionScript (SimArgs 1s 6) (Script ((fromList [("test1",[(3d:482c:fa:20de:c0:ff3e:37:246b,300)])],NoDelay) :| [(fromList [("test1",[(83:e0e9:df:1988:88:bdd0:a1:9113,300)]),("test4",[(96:d4f2:9b:2a0:cc:2c10:5f:fa51,300)])],LongDelay)])) [(NodeArgs (17) InitiatorAndResponderDiffusionMode (Just 90s) (fromList [(RelayAccessDomain "test1" 34,DoAdvertisePeer),(RelayAccessAddress "2a:541b:b:64c0:28:8a37:5e:73d6" 18,DoAdvertisePeer)]) (TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)) NoPeerSharing [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test1" 34,DoNotAdvertisePeer),(RelayAccessAddress "2a:541b:b:64c0:28:8a37:5e:73d6" 18,DoNotAdvertisePeer)])] PeerSelectionTargets {targetNumberOfRootPeers = 2, targetNumberOfKnownPeers = 3, targetNumberOfEstablishedPeers = 2, targetNumberOfActivePeers = 2, targetNumberOfKnownBigLedgerPeers = 6, targetNumberOfEstablishedBigLedgerPeers = 2, targetNumberOfActiveBigLedgerPeers = 2} (Script (DNSTimeout {getDNSTimeout = 0.182s} :| [DNSTimeout {getDNSTimeout = 0.384s},DNSTimeout {getDNSTimeout = 0.349s},DNSTimeout {getDNSTimeout = 0.191s},DNSTimeout {getDNSTimeout = 0.164s},DNSTimeout {getDNSTimeout = 0.323s},DNSTimeout {getDNSTimeout = 0.22s},DNSTimeout {getDNSTimeout = 0.18s},DNSTimeout {getDNSTimeout = 0.322s},DNSTimeout {getDNSTimeout = 0.135s},DNSTimeout {getDNSTimeout = 0.3s},DNSTimeout {getDNSTimeout = 0.247s},DNSTimeout {getDNSTimeout = 0.197s},DNSTimeout {getDNSTimeout = 0.35s},DNSTimeout {getDNSTimeout = 0.309s},DNSTimeout {getDNSTimeout = 0.283s},DNSTimeout {getDNSTimeout = 0.293s},DNSTimeout {getDNSTimeout = 0.262s},DNSTimeout {getDNSTimeout = 0.22s},DNSTimeout {getDNSTimeout = 0.281s},DNSTimeout {getDNSTimeout = 0.262s},DNSTimeout {getDNSTimeout = 0.134s},DNSTimeout {getDNSTimeout = 0.357s},DNSTimeout {getDNSTimeout = 0.196s},DNSTimeout {getDNSTimeout = 0.384s},DNSTimeout {getDNSTimeout = 0.184s},DNSTimeout {getDNSTimeout = 0.374s},DNSTimeout {getDNSTimeout = 0.396s},DNSTimeout {getDNSTimeout = 0.287s},DNSTimeout {getDNSTimeout = 0.323s},DNSTimeout {getDNSTimeout = 0.136s},DNSTimeout {getDNSTimeout = 0.188s},DNSTimeout {getDNSTimeout = 0.127s}])) (Script (DNSLookupDelay {getDNSLookupDelay = 0.107s} :| [DNSLookupDelay {getDNSLookupDelay = 0.047s},DNSLookupDelay {getDNSLookupDelay = 0.025s},DNSLookupDelay {getDNSLookupDelay = 0.059s},DNSLookupDelay {getDNSLookupDelay = 0.129s},DNSLookupDelay {getDNSLookupDelay = 0.072s},DNSLookupDelay {getDNSLookupDelay = 0.069s},DNSLookupDelay {getDNSLookupDelay = 0.035s},DNSLookupDelay {getDNSLookupDelay = 0.127s},DNSLookupDelay {getDNSLookupDelay = 0.11s},DNSLookupDelay {getDNSLookupDelay = 0.039s},DNSLookupDelay {getDNSLookupDelay = 0.024s},DNSLookupDelay {getDNSLookupDelay = 0.039s},DNSLookupDelay {getDNSLookupDelay = 0.113s},DNSLookupDelay {getDNSLookupDelay = 0.05s},DNSLookupDelay {getDNSLookupDelay = 0.096s},DNSLookupDelay {getDNSLookupDelay = 0.033s},DNSLookupDelay {getDNSLookupDelay = 0.042s},DNSLookupDelay {getDNSLookupDelay = 0.096s},DNSLookupDelay {getDNSLookupDelay = 0.048s},DNSLookupDelay {getDNSLookupDelay = 0.094s},DNSLookupDelay {getDNSLookupDelay = 0.064s},DNSLookupDelay {getDNSLookupDelay = 0.049s},DNSLookupDelay {getDNSLookupDelay = 0.022s},DNSLookupDelay {getDNSLookupDelay = 0.047s},DNSLookupDelay {getDNSLookupDelay = 0.063s},DNSLookupDelay {getDNSLookupDelay = 0.074s},DNSLookupDelay {getDNSLookupDelay = 0.097s},DNSLookupDelay {getDNSLookupDelay = 0.111s}])) (Just (BlockNo 3)) False,[JoinNetwork 99.19540229885s,Reconfigure 377.17647058823s [],Reconfigure 3.35s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test1" 34,DoNotAdvertisePeer),(RelayAccessAddress "2a:541b:b:64c0:28:8a37:5e:73d6" 18,DoNotAdvertisePeer)])],Reconfigure 340s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test1" 34,DoNotAdvertisePeer),(RelayAccessAddress "2a:541b:b:64c0:28:8a37:5e:73d6" 18,DoNotAdvertisePeer)])],Kill 17.315789473684s,JoinNetwork 120.5s,Kill 3.032967032967s,JoinNetwork 2.973170731707s,Reconfigure 8.986842105263s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test1" 34,DoNotAdvertisePeer)])],Reconfigure 1.142857142857s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [])],Reconfigure 464.83333333333s [],Reconfigure 2.915384615384s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessAddress "2a:541b:b:64c0:28:8a37:5e:73d6" 18,DoNotAdvertisePeer)])],Kill 1.232183908045s,JoinNetwork 3.129787234042s,Kill 3.601960784313s,JoinNetwork 395.625s,Kill 16.7s,JoinNetwork 2.45352112676s,Kill 17.23684210526s,JoinNetwork 1.426829268292s]),(NodeArgs (31) InitiatorAndResponderDiffusionMode (Just 180s) (fromList [(RelayAccessDomain "test1" 34,DoAdvertisePeer),(RelayAccessDomain "test4" 9,DoAdvertisePeer)]) (TestAddress (IPAddr (read "2a:541b:b:64c0:28:8a37:5e:73d6") 18)) PeerSharingPublic [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test1" 34,DoNotAdvertisePeer),(RelayAccessDomain "test4" 9,DoNotAdvertisePeer)])] PeerSelectionTargets {targetNumberOfRootPeers = 4, targetNumberOfKnownPeers = 4, targetNumberOfEstablishedPeers = 4, targetNumberOfActivePeers = 4, targetNumberOfKnownBigLedgerPeers = 3, targetNumberOfEstablishedBigLedgerPeers = 0, targetNumberOfActiveBigLedgerPeers = 0} (Script (DNSTimeout {getDNSTimeout = 0.2s} :| [DNSTimeout {getDNSTimeout = 0.171s},DNSTimeout {getDNSTimeout = 0.382s},DNSTimeout {getDNSTimeout = 0.407s},DNSTimeout {getDNSTimeout = 0.276s},DNSTimeout {getDNSTimeout = 0.201s},DNSTimeout {getDNSTimeout = 0.336s},DNSTimeout {getDNSTimeout = 0.346s},DNSTimeout {getDNSTimeout = 0.39s},DNSTimeout {getDNSTimeout = 0.165s},DNSTimeout {getDNSTimeout = 0.352s},DNSTimeout {getDNSTimeout = 0.332s},DNSTimeout {getDNSTimeout = 0.16s},DNSTimeout {getDNSTimeout = 0.319s}])) (Script (DNSLookupDelay {getDNSLookupDelay = 0.097s} :| [])) (Nothing) False,[JoinNetwork 3.2875s,Reconfigure 34s [],Kill 171.81818181818s,JoinNetwork 2.083333333333s,Kill 90s,JoinNetwork 415s,Reconfigure 31.571428571428s [],Reconfigure 13.803921568627s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test1" 34,DoNotAdvertisePeer)])],Kill 44.16923076923s,JoinNetwork 0.05s,Reconfigure 1.571604938271s [],Reconfigure 32.802631578947s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test1" 34,DoNotAdvertisePeer)])],Kill 331.89189189189s,JoinNetwork 17.90909090909s,Reconfigure 80.55555555555s [],Reconfigure 24.692307692307s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test4" 9,DoNotAdvertisePeer)])],Reconfigure 161.46341463414s [],Reconfigure 0.571764705882s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test1" 34,DoNotAdvertisePeer)])],Reconfigure 0.95s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [])],Reconfigure 0.8s [],Reconfigure 13.125s []]),(NodeArgs (-33) InitiatorAndResponderDiffusionMode (Just 180s) (fromList [(RelayAccessDomain "test4" 9,DoAdvertisePeer),(RelayAccessAddress "2a:541b:b:64c0:28:8a37:5e:73d6" 18,DoAdvertisePeer)]) (TestAddress (IPAddr (read "83:e0e9:df:1988:88:bdd0:a1:9113") 34)) NoPeerSharing [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test4" 9,DoNotAdvertisePeer),(RelayAccessAddress "2a:541b:b:64c0:28:8a37:5e:73d6" 18,DoAdvertisePeer)])] PeerSelectionTargets {targetNumberOfRootPeers = 2, targetNumberOfKnownPeers = 6, targetNumberOfEstablishedPeers = 3, targetNumberOfActivePeers = 1, targetNumberOfKnownBigLedgerPeers = 5, targetNumberOfEstablishedBigLedgerPeers = 4, targetNumberOfActiveBigLedgerPeers = 3} (Script (DNSTimeout {getDNSTimeout = 0.214s} :| [DNSTimeout {getDNSTimeout = 0.373s},DNSTimeout {getDNSTimeout = 0.326s},DNSTimeout {getDNSTimeout = 0.208s},DNSTimeout {getDNSTimeout = 0.248s},DNSTimeout {getDNSTimeout = 0.172s},DNSTimeout {getDNSTimeout = 0.307s},DNSTimeout {getDNSTimeout = 0.313s},DNSTimeout {getDNSTimeout = 0.3s},DNSTimeout {getDNSTimeout = 0.12s},DNSTimeout {getDNSTimeout = 0.178s},DNSTimeout {getDNSTimeout = 0.2s},DNSTimeout {getDNSTimeout = 0.156s},DNSTimeout {getDNSTimeout = 0.161s},DNSTimeout {getDNSTimeout = 0.194s},DNSTimeout {getDNSTimeout = 0.41s},DNSTimeout {getDNSTimeout = 0.369s}])) (Script (DNSLookupDelay {getDNSLookupDelay = 0.069s} :| [DNSLookupDelay {getDNSLookupDelay = 0.057s},DNSLookupDelay {getDNSLookupDelay = 0.075s},DNSLookupDelay {getDNSLookupDelay = 0.095s},DNSLookupDelay {getDNSLookupDelay = 0.048s},DNSLookupDelay {getDNSLookupDelay = 0.137s},DNSLookupDelay {getDNSLookupDelay = 0.066s},DNSLookupDelay {getDNSLookupDelay = 0.064s},DNSLookupDelay {getDNSLookupDelay = 0.064s},DNSLookupDelay {getDNSLookupDelay = 0.064s},DNSLookupDelay {getDNSLookupDelay = 0.136s},DNSLookupDelay {getDNSLookupDelay = 0.043s},DNSLookupDelay {getDNSLookupDelay = 0.027s},DNSLookupDelay {getDNSLookupDelay = 0.071s},DNSLookupDelay {getDNSLookupDelay = 0.134s},DNSLookupDelay {getDNSLookupDelay = 0.116s},DNSLookupDelay {getDNSLookupDelay = 0.058s},DNSLookupDelay {getDNSLookupDelay = 0.072s},DNSLookupDelay {getDNSLookupDelay = 0.047s},DNSLookupDelay {getDNSLookupDelay = 0.029s},DNSLookupDelay {getDNSLookupDelay = 0.053s},DNSLookupDelay {getDNSLookupDelay = 0.032s},DNSLookupDelay {getDNSLookupDelay = 0.037s},DNSLookupDelay {getDNSLookupDelay = 0.106s},DNSLookupDelay {getDNSLookupDelay = 0.047s},DNSLookupDelay {getDNSLookupDelay = 0.127s},DNSLookupDelay {getDNSLookupDelay = 0.025s},DNSLookupDelay {getDNSLookupDelay = 0.043s},DNSLookupDelay {getDNSLookupDelay = 0.063s},DNSLookupDelay {getDNSLookupDelay = 0.124s},DNSLookupDelay {getDNSLookupDelay = 0.093s},DNSLookupDelay {getDNSLookupDelay = 0.086s},DNSLookupDelay {getDNSLookupDelay = 0.123s},DNSLookupDelay {getDNSLookupDelay = 0.092s},DNSLookupDelay {getDNSLookupDelay = 0.126s},DNSLookupDelay {getDNSLookupDelay = 0.05s},DNSLookupDelay {getDNSLookupDelay = 0.024s},DNSLookupDelay {getDNSLookupDelay = 0.055s},DNSLookupDelay {getDNSLookupDelay = 0.13s},DNSLookupDelay {getDNSLookupDelay = 0.042s}])) (Just (BlockNo 2)) False,[JoinNetwork 417.30769230769s,Reconfigure 461.25s [],Reconfigure 6.076923076923s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [])],Reconfigure 2.819565217391s [],Reconfigure 19.157894736842s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test4" 9,DoNotAdvertisePeer)])],Kill 191.91489361702s,JoinNetwork 36.384615384615s,Reconfigure 9.197530864197s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [])],Kill 13.067567567567s,JoinNetwork 1.298863636363s,Reconfigure 143.33333333333s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test4" 9,DoNotAdvertisePeer),(RelayAccessAddress "2a:541b:b:64c0:28:8a37:5e:73d6" 18,DoAdvertisePeer)])],Kill 4.138372093023s,JoinNetwork 230.4s,Reconfigure 318.66666666666s [],Reconfigure 37.695652173913s [(HotValency {getHotValency = 2},WarmValency {getWarmValency = 2},fromList [(RelayAccessDomain "test4" 9,DoNotAdvertisePeer)])],Kill 0.08064516129s,JoinNetwork 55.55555555555s,Kill 13s,JoinNetwork 343.18181818181s])]

...
            
      Last 20 signal values:
      Time 0s   : fromList []
      
      Property violated at: Time 788.407157215751s
      Invalid signal value:
      fromList [TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)]
      
      Use --quickcheck-replay=317002 to reproduce.
      Use -p '$0=="ouroboros-network:sim-tests.Ouroboros.Network.Testnet.async demotion"' to rerun this test only.

The short section of the trace:

      (Time 787.407157215751s,DiffusionPeerSelectionActionsTrace (PeerMonitoringResult (ConnectionId {localAddress = TestAddress (IPAddr (read "2a:541b:b:64c0:28:8a37:5e:73d6") 18), remoteAddress = TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)}) (WithSomeProtocolTemperature (WithHot (MiniProtocolError (MiniProtocolException {mpeMiniProtocolNumber = MiniProtocolNum 2, mpeMiniProtocolException = ExceededTimeLimit (ChainSync (BlockHeader) (Tip Block)) (ServerAgency TokNext TokCanAwait)}))))))
      (Time 787.407157215751s,DiffusionPeerSelectionActionsTrace (PeerStatusChanged (HotToCold (ConnectionId {localAddress = TestAddress (IPAddr (read "2a:541b:b:64c0:28:8a37:5e:73d6") 18), remoteAddress = TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)}))))
      (Time 787.407157215751s,DiffusionInboundGovernorTrace (TrMuxErrored (ConnectionId {localAddress = TestAddress (IPAddr (read "2a:541b:b:64c0:28:8a37:5e:73d6") 18), remoteAddress = TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)}) ExceededTimeLimit (ChainSync (BlockHeader) (Tip Block)) (ServerAgency TokNext TokCanAwait)))
      (Time 787.407157215751s,DiffusionInboundGovernorTransitionTrace TransitionTrace @(TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)) (Just RemoteColdSt → Nothing))
      (Time 787.407157215751s,DiffusionPeerSelectionActionsTrace (PeerStatusChangeFailure (HotToCold (ConnectionId {localAddress = TestAddress (IPAddr (read "2a:541b:b:64c0:28:8a37:5e:73d6") 18), remoteAddress = TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)})) (ApplicationFailure [MiniProtocolException {mpeMiniProtocolNumber = MiniProtocolNum 2, mpeMiniProtocolException = ExceededTimeLimit (ChainSync (BlockHeader) (Tip Block)) (ServerAgency TokNext TokCanAwait)},MiniProtocolException {mpeMiniProtocolNumber = MiniProtocolNum 3, mpeMiniProtocolException = MuxError (MuxShutdown Nothing) "ExceededTimeLimit (ChainSync (BlockHeader) (Tip Block)) (ServerAgency TokNext TokCanAwait)"}])))
      (Time 787.407157215751s,DiffusionInboundGovernorTrace (TrRemoteState (fromList [])))
      (Time 787.407157215751s,DiffusionPeerSelectionTrace (TraceDemoteHotBigLedgerPeerFailed 0 1 TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9) (MiniProtocolExceptions [MiniProtocolException {mpeMiniProtocolNumber = MiniProtocolNum 2, mpeMiniProtocolException = ExceededTimeLimit (ChainSync (BlockHeader) (Tip Block)) (ServerAgency TokNext TokCanAwait)},MiniProtocolException {mpeMiniProtocolNumber = MiniProtocolNum 3, mpeMiniProtocolException = MuxError (MuxShutdown Nothing) "ExceededTimeLimit (ChainSync (BlockHeader) (Tip Block)) (ServerAgency TokNext TokCanAwait)"}])))
      (Time 787.407157215751s,DiffusionDebugPeerSelectionTrace (TraceGovernorState (Time 787.407157215751s) (Just 9.079094446986s) (PeerSelectionState {targets = PeerSelectionTargets {targetNumberOfRootPeers = 4, targetNumberOfKnownPeers = 4, targetNumberOfEstablishedPeers = 4, targetNumberOfActivePeers = 4, targetNumberOfKnownBigLedgerPeers = 3, targetNumberOfEstablishedBigLedgerPeers = 0, targetNumberOfActiveBigLedgerPeers = 0}, localRootPeers = fromGroups [(HotValency {getHotValency = 1},WarmValency {getWarmValency = 1},fromList [(TestAddress (IPAddr (read "83:e0e9:df:1988:88:bdd0:a1:9113") 34),DoNotAdvertisePeer)])], publicRootPeers = fromList [], bigLedgerPeers = fromList [TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)], knownPeers = KnownPeers {allPeers = fromList [(TestAddress (IPAddr (read "83:e0e9:df:1988:88:bdd0:a1:9113") 34),KnownPeerInfo {knownPeerFailCount = 1, knownPeerTepid = False, knownPeerSharing = PeerSharingPrivate, knownPeerAdvertise = DoNotAdvertisePeer, knownLedgerPeer = IsNotLedgerPeer}),(TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9),KnownPeerInfo {knownPeerFailCount = 1, knownPeerTepid = False, knownPeerSharing = NoPeerSharing, knownPeerAdvertise = DoNotAdvertisePeer, knownLedgerPeer = IsLedgerPeer})], availableToConnect = fromList [], nextConnectTimes = Winner (E TestAddress (IPAddr (read "83:e0e9:df:1988:88:bdd0:a1:9113") 34) (Time 796.486251662737s) ()) (RLoser 1 (E TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9) (Time 798.835808965814s) ()) Start TestAddress (IPAddr (read "83:e0e9:df:1988:88:bdd0:a1:9113") 34) Start) TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)}, establishedPeers = EstablishedPeers {allPeers = fromList [], availableForPeerShare = fromList [], nextPeerShareTimes = Void, nextActivateTimes = Void}, activePeers = fromList [], publicRootBackoffs = 6, publicRootRetryTime = Time 842.771015151513s, inProgressPublicRootsReq = False, bigLedgerPeerBackoffs = 0, bigLedgerPeerRetryTime = Time 1078.771015151513s, inProgressBigLedgerPeersReq = False, inProgressPeerShareReqs = 0, inProgressPromoteCold = fromList [], inProgressPromoteWarm = fromList [], inProgressDemoteWarm = fromList [], inProgressDemoteHot = fromList [], fuzzRng = StdGen {unStdGen = SMGen 9422704847422864037 10345829224351529773}, countersCache = Cache {getCache = PeerSelectionCounters {coldPeers = 1, warmPeers = 0, hotPeers = 0, coldBigLedgerPeers = 1, warmBigLedgerPeers = 0, hotBigLedgerPeers = 0, localRoots = [(0,0)]}}})))
      (Time 787.407157215751s,DiffusionConnectionManagerTrace (TrConnectionHandler (ConnectionId {localAddress = TestAddress (IPAddr (read "2a:541b:b:64c0:28:8a37:5e:73d6") 18), remoteAddress = TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)}) (TrConnectionHandlerError OutboundError ExceededTimeLimit (ChainSync (BlockHeader) (Tip Block)) (ServerAgency TokNext TokCanAwait) ShutdownPeer)))
      (Time 787.407157215751s,DiffusionConnectionManagerTrace (TrConnectionCleanup (ConnectionId {localAddress = TestAddress (IPAddr (read "2a:541b:b:64c0:28:8a37:5e:73d6") 18), remoteAddress = TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)})))
      (Time 787.407157215751s,DiffusionConnectionManagerTransitionTrace TransitionTrace @(TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)) (OutboundDupSt Expired → TerminatedSt))
      (Time 787.407157215751s,DiffusionConnectionManagerTransitionTrace TransitionTrace @(TestAddress (IPAddr (read "96:d4f2:9b:2a0:cc:2c10:5f:fa51") 9)) (TerminatedSt → UnknownConnectionSt))
      (Time 787.407157215751s,DiffusionConnectionManagerTrace (TrConnectionManagerCounters (ConnectionManagerCounters {fullDuplexConns = 0, duplexConns = 0, unidirectionalConns = 0, inboundConns = 0, outboundConns = 0})))
      (Time 796.486251662737s,DiffusionPeerSelectionTrace TraceGovernorWakeup)

Whole trace is available here.

[coot]

Another case: https://ci.iog.io/build/723516/nixlog/1