Improved network halted handling for case when we cannot connect to any

top tier nodes, and a batch of connected peer nodes don't relay
top tier externalize messages because they are stuck. If a next batch
detects a ledger close, the network is not marked as halted

src/network-observer/__tests__/observation-manager.test.ts

@@ -120,11 +120,12 @@ describe('ObservationManager', () => {
 			new Slots(new QuorumSet(1, ['A'], []), mock<P.Logger>())
 		);
 		await observationManager.startSync(observation);
-		expect(observation.networkHalted).toBe(false);
+		expect(observation.isNetworkHalted()).toBe(false);
 		const networkHaltedCallback = consensusTimer.start.mock
 			.calls[0][0] as () => void;
 		networkHaltedCallback();
-		expect(observation.networkHalted).toBe(true);
+		expect(observation.isNetworkHalted()).toBe(true);
+
 		expect(
 			stragglerTimer.startStragglerTimeoutForActivePeers
 		).toHaveBeenCalled();

src/network-observer/__tests__/observation.test.ts

@@ -87,18 +87,19 @@ describe('Observation', () => {
 		expect(observation.latestConfirmedClosedLedger.sequence).toBe(BigInt(0));
 	});
 
-	it('should not confirm ledger close if network halted', () => {
+	it('should mark network halted if new ledger is found after network was previously halted', () => {
 		const observation = createObservation();
 		observation.moveToSyncingState();
 		observation.moveToSyncedState();
-		observation.networkHalted = true;
+		observation.setNetworkHalted();
 		const ledger = {
 			sequence: BigInt(1),
 			closeTime: new Date(),
 			value: 'value',
 			localCloseTime: new Date()
 		};
 		observation.ledgerCloseConfirmed(ledger);
-		expect(observation.latestConfirmedClosedLedger.sequence).toBe(BigInt(0));
+		expect(observation.latestConfirmedClosedLedger.sequence).toBe(BigInt(1));
+		expect(observation.isNetworkHalted()).toBeFalsy();
 	});
 });

src/network-observer/observation-manager.ts

@@ -54,7 +54,7 @@ export class ObservationManager {
 
 	private onNetworkHalted(observation: Observation) {
 		this.logger.info('Network consensus timeout');
-		observation.networkHalted = true;
+		observation.setNetworkHalted();
 		this.stragglerTimer.startStragglerTimeoutForActivePeers(
 			false,
 			observation.topTierAddressesSet

src/network-observer/observation.ts

@@ -10,7 +10,7 @@ import { QuorumSetState } from './quorum-set-state';
 
 export class Observation {
 	public state: ObservationState = ObservationState.Idle;
-	public networkHalted = false;
+	private networkHalted = false;
 	public topTierAddressesSet: Set<string>;
 	public envelopeCache: LRUCache<string, number>;
 	public quorumSetState: QuorumSetState = new QuorumSetState();
@@ -56,9 +56,17 @@ export class Observation {
 	}
 
 	ledgerCloseConfirmed(ledger: Ledger) {
+		this.networkHalted = false;
 		if (this.state !== ObservationState.Synced) return;
-		if (this.networkHalted) return;
 
 		this.latestConfirmedClosedLedger = ledger;
 	}
+
+	isNetworkHalted(): boolean {
+		return this.networkHalted;
+	}
+
+	setNetworkHalted() {
+		this.networkHalted = true;
+	}
 }

src/network-observer/peer-event-handler/__tests__/on-peer-connected.test.ts

@@ -128,7 +128,7 @@ describe('OnPeerConnectedHandler', () => {
 		const data = createData();
 		const observation = createObservation();
 		observation.state = ObservationState.Synced;
-		observation.networkHalted = true;
+		observation.setNetworkHalted();
 		onConnectedHandler.handle(data, observation);
 		assertPeerSuccessfullyConnected(observation.peerNodes, data);
 		assertStragglerTimeoutStarted(data);

src/network-observer/peer-event-handler/on-peer-connected.ts

@@ -21,7 +21,7 @@ export class OnPeerConnected {
 			return peerNodeOrError;
 		}
 
-		if (this.networkIsHalted(observation)) {
+		if (observation.isNetworkHalted()) {
 			return this.collectMinimalDataAndDisconnect(data);
 		}
 
@@ -53,11 +53,6 @@ export class OnPeerConnected {
 	private collectMinimalDataAndDisconnect(data: ConnectedPayload) {
 		return this.startStragglerTimeout(data);
 	}
-
-	private networkIsHalted(observation: Observation) {
-		return observation.networkHalted;
-	}
-
 	private startStragglerTimeout(data: ConnectedPayload) {
 		return this.stragglerHandler.startStragglerTimeout([
 			data.ip + ':' + data.port