refactored crawl-state to crawl. Moved properties to Observation when

needed.

.eslintignore

@@ -1,9 +1,6 @@
 # don't ever lint node_modules
 node_modules
-# don't lint build output (make sure it's set to your correct build folder name)
-dist
 # don't lint nyc coverage output
 coverage
 examples
-lib
-node_modules
+lib

src/__fixtures__/createDummyCrawlerConfiguration.ts

@@ -1,6 +1,5 @@
 import { CrawlerConfiguration } from '../crawler-configuration';
 import { NodeConfig } from '@stellarbeat/js-stellar-node-connector/lib/node-config';
-import { NodeInfo } from '@stellarbeat/js-stellar-node-connector/lib/node';
 
 export function createDummyCrawlerConfiguration(): CrawlerConfiguration {
 	const nodeConfig: NodeConfig = {

src/__tests__/crawl-queue-manager.test.ts

@@ -2,14 +2,14 @@ import { AsyncCrawlQueue } from '../crawl-queue';
 import { CrawlQueueManager } from '../crawl-queue-manager';
 import { mock } from 'jest-mock-extended';
 import { P } from 'pino';
-import { CrawlState } from '../crawl-state';
+import { Crawl } from '../crawl';
 import { CrawlTask } from '../crawl-task';
 import { nodeAddressToPeerKey } from '../node-address';
 
 describe('CrawlQueueManager', () => {
 	const crawlQueue = mock<AsyncCrawlQueue>();
 	const logger = mock<P.Logger>();
-	const crawlState = mock<CrawlState>();
+	const crawlState = mock<Crawl>();
 
 	beforeEach(() => {
 		crawlState.crawledNodeAddresses = new Set();
@@ -21,7 +21,7 @@ describe('CrawlQueueManager', () => {
 		const crawlQueueManager = new CrawlQueueManager(crawlQueue, logger);
 		crawlQueueManager.addCrawlTask({
 			connectCallback: () => {},
-			crawlState,
+			crawl: crawlState,
 			nodeAddress: ['localhost', 11625]
 		});
 
@@ -48,7 +48,7 @@ describe('CrawlQueueManager', () => {
 	it('should perform a crawl queue task', () => {
 		const task: CrawlTask = {
 			connectCallback: jest.fn(),
-			crawlState,
+			crawl: crawlState,
 			nodeAddress: ['localhost', 11625]
 		};
 
@@ -64,7 +64,7 @@ describe('CrawlQueueManager', () => {
 	it('should complete a crawl task', function () {
 		const task: CrawlTask = {
 			connectCallback: jest.fn(),
-			crawlState,
+			crawl: crawlState,
 			nodeAddress: ['localhost', 11625]
 		};
 

src/__tests__/crawler.integration.test.ts

@@ -12,9 +12,10 @@ import { ok, Result, err } from 'neverthrow';
 import { CrawlerConfiguration, createCrawler } from '../index';
 import { StellarMessageWork } from '@stellarbeat/js-stellar-node-connector/lib/connection/connection';
 import { NodeAddress } from '../node-address';
-import { CrawlState } from '../crawl-state';
 import { mock } from 'jest-mock-extended';
 import { P } from 'pino';
+import { CrawlFactory } from '../crawl-factory';
+import { ObservationFactory } from '../network-observer/observation-factory';
 
 jest.setTimeout(60000);
 
@@ -168,24 +169,23 @@ it('should crawl, listen for validating nodes and harvest quorumSets', async ()
 	crawlerConfig.syncingTimeoutMS = 100;
 	crawlerConfig.quorumSetRequestTimeoutMS = 100;
 	const crawler = createCrawler(crawlerConfig);
-	const crawlState = new CrawlState(
+	const crawlerFactory = new CrawlFactory(new ObservationFactory());
+	const crawl = crawlerFactory.createCrawl(
+		nodeConfig.network,
+		[peerNodeAddress, publicKeyReusingPeerNodeAddress],
+		[],
 		trustedQSet,
-		new Map(),
 		{
 			sequence: BigInt(0),
 			closeTime: new Date(0),
 			value: '',
 			localCloseTime: new Date(0)
 		},
-		nodeConfig.network,
+		new Map<string, QuorumSet>(),
 		mock<P.Logger>()
 	);
 
-	const result = await crawler.crawl(
-		[peerNodeAddress, publicKeyReusingPeerNodeAddress],
-		[],
-		crawlState
-	);
+	const result = await crawler.startCrawl(crawl);
 	const peerNode = result.peers.get(peerNetworkNode.keyPair.publicKey());
 	expect(peerNode).toBeDefined();
 	if (!peerNode) return;
@@ -223,27 +223,24 @@ it('should hit the max crawl limit', async function () {
 	const crawler = createCrawler(
 		new CrawlerConfiguration(nodeConfig, 25, 1000, new Set(), 1000, 100, 100)
 	);
-	const crawlState = new CrawlState(
+	const crawlerFactory = new CrawlFactory(new ObservationFactory());
+	const crawl = crawlerFactory.createCrawl(
+		nodeConfig.network,
+		[peerNodeAddress, publicKeyReusingPeerNodeAddress],
+		[],
 		trustedQSet,
-		new Map(),
 		{
 			sequence: BigInt(0),
 			closeTime: new Date(0),
 			value: '',
 			localCloseTime: new Date(0)
 		},
-		nodeConfig.network,
+		new Map<string, QuorumSet>(),
 		mock<P.Logger>()
 	);
 
 	try {
-		expect(
-			await crawler.crawl(
-				[peerNodeAddress, publicKeyReusingPeerNodeAddress],
-				[],
-				crawlState
-			)
-		).toThrowError();
+		expect(await crawler.startCrawl(crawl)).toThrowError();
 	} catch (e) {
 		expect(e).toBeInstanceOf(Error);
 	}

src/__tests__/crawler.test.ts

@@ -6,15 +6,17 @@ import { P } from 'pino';
 import { mock, MockProxy } from 'jest-mock-extended';
 import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared';
 import { CrawlLogger } from '../crawl-logger';
-import { CrawlProcessState, CrawlState } from '../crawl-state';
+import { CrawlProcessState } from '../crawl';
 import { EventEmitter } from 'events';
 import { AsyncCrawlQueue } from '../crawl-queue';
 import { NetworkObserver } from '../network-observer/network-observer';
 import { ClosePayload } from '../network-observer/connection-manager';
+import { ObservationFactory } from '../network-observer/observation-factory';
+import { CrawlFactory } from '../crawl-factory';
 import { Observation } from '../network-observer/observation';
-import { PeerNodeCollection } from '../peer-node-collection';
 
 describe('Crawler', () => {
+	const crawlFactory = new CrawlFactory(new ObservationFactory());
 	beforeEach(() => {
 		jest.clearAllMocks();
 	});
@@ -44,22 +46,24 @@ describe('Crawler', () => {
 			crawlLogger,
 			logger
 		);
-		const crawlState = new CrawlState(
+		const crawl = crawlFactory.createCrawl(
+			'test',
+			[['peer', 2]],
+			[['top', 1]],
 			new QuorumSet(2, []),
-			new Map(),
 			{
 				closeTime: new Date(0),
 				localCloseTime: new Date(0),
 				sequence: BigInt(0),
 				value: ''
 			},
-			'test',
+			new Map(),
 			logger
 		);
 
 		return {
 			crawler,
-			crawlState,
+			crawl,
 			networkObserver,
 			networkObserverEventEmitter,
 			crawlLogger,
@@ -75,14 +79,16 @@ describe('Crawler', () => {
 	it('should return error if no active top tier connections and no node addresses to crawl', async () => {
 		const {
 			crawler,
-			crawlState,
+			crawl: crawl,
 			networkObserver,
 			crawlLogger,
 			maxCrawlTimeManager
 		} = setupSUT();
-		networkObserver.observe.mockResolvedValue(0);
+		networkObserver.startObservation.mockResolvedValue(0);
+		crawl.observation.topTierAddresses = [];
+		crawl.nodesToCrawl = [];
 		try {
-			await crawler.crawl([], [], crawlState);
+			await crawler.startCrawl(crawl);
 		} catch (e) {
 			expect(e).toBeInstanceOf(Error);
 			expect(crawlLogger.start).not.toHaveBeenCalled();
@@ -107,16 +113,16 @@ describe('Crawler', () => {
 	it('should connect to top tier and not crawl if there are no nodes to be crawled', async () => {
 		const {
 			crawler,
-			crawlState,
+			crawl: crawl,
 			networkObserver,
 			crawlLogger,
 			maxCrawlTimeManager
 		} = setupSUT();
-		networkObserver.observe.mockResolvedValue(1);
-		networkObserver.stop.mockResolvedValue(
-			new Observation([], mock<PeerNodeCollection>(), mock<CrawlState>())
-		);
-		const result = await crawler.crawl([], [], crawlState);
+		networkObserver.startObservation.mockResolvedValue(1);
+		networkObserver.stop.mockResolvedValue(mock<Observation>());
+		crawl.observation.topTierAddresses = [];
+		crawl.nodesToCrawl = [];
+		const result = await crawler.startCrawl(crawl);
 		expect(result).toEqual({
 			closedLedgers: [],
 			latestClosedLedger: {
@@ -135,20 +141,23 @@ describe('Crawler', () => {
 	it('should connect to top tier and crawl peer nodes received from top tier', (resolve) => {
 		const {
 			crawler,
-			crawlState,
+			crawl: crawl,
 			networkObserver,
 			networkObserverEventEmitter,
 			crawlLogger,
 			maxCrawlTimeManager
 		} = setupSUT();
-		networkObserver.observe.mockImplementationOnce(() => {
+		networkObserver.startObservation.mockImplementationOnce(() => {
 			return new Promise((resolve) => {
 				networkObserverEventEmitter.emit('peers', [['127.0.0.1', 11625]]);
 				setTimeout(() => {
 					resolve(1);
 				}, 1);
 			});
 		});
+
+		networkObserver.stop.mockResolvedValue(mock<Observation>());
+
 		networkObserver.connectToNode.mockImplementation((address, port) => {
 			return new Promise((resolve) => {
 				const disconnectPayload: ClosePayload = {
@@ -162,11 +171,8 @@ describe('Crawler', () => {
 			});
 		});
 
-		networkObserver.stop.mockResolvedValue(
-			new Observation([], mock<PeerNodeCollection>(), mock<CrawlState>())
-		);
 		crawler
-			.crawl([['peer', 2]], [['top', 1]], crawlState)
+			.startCrawl(crawl)
 			.then((result) => {
 				expect(result).toEqual({
 					closedLedgers: [],
@@ -180,13 +186,12 @@ describe('Crawler', () => {
 				});
 				expectCorrectMaxTimer(maxCrawlTimeManager);
 				expectCorrectLogger(crawlLogger);
-				expect(networkObserver.observe).toHaveBeenNthCalledWith(
+				expect(networkObserver.startObservation).toHaveBeenNthCalledWith(
 					1,
-					[['top', 1]],
-					crawlState
+					crawl.observation
 				);
 				expect(networkObserver.connectToNode).toHaveBeenCalledTimes(2);
-				expect(crawlState.state).toBe(CrawlProcessState.IDLE);
+				expect(crawl.state).toBe(CrawlProcessState.IDLE);
 				resolve();
 			})
 			.catch((e) => {
@@ -197,16 +202,14 @@ describe('Crawler', () => {
 	it('should crawl nodes received from peers', (resolve) => {
 		const {
 			crawler,
-			crawlState,
+			crawl,
 			networkObserver,
 			crawlLogger,
 			maxCrawlTimeManager,
 			networkObserverEventEmitter
 		} = setupSUT();
-		networkObserver.observe.mockResolvedValue(1);
-		networkObserver.stop.mockResolvedValue(
-			new Observation([], mock<PeerNodeCollection>(), mock<CrawlState>())
-		);
+		networkObserver.startObservation.mockResolvedValue(1);
+		networkObserver.stop.mockResolvedValue(mock<Observation>());
 		networkObserver.connectToNode.mockImplementation((address, port) => {
 			return new Promise((resolve) => {
 				const disconnectPayload: ClosePayload = {
@@ -221,7 +224,7 @@ describe('Crawler', () => {
 			});
 		});
 		crawler
-			.crawl([['peer', 2]], [['top', 1]], crawlState)
+			.startCrawl(crawl)
 			.then((result) => {
 				expect(result).toEqual({
 					closedLedgers: [],

src/crawl-factory.ts

@@ -0,0 +1,31 @@
+import { Crawl } from './crawl';
+import { ObservationFactory } from './network-observer/observation-factory';
+import { Slots } from './network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slots';
+import { NodeAddress } from './node-address';
+import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared';
+import { Ledger } from './crawler';
+import { P } from 'pino';
+import { PeerNodeCollection } from './peer-node-collection';
+
+export class CrawlFactory {
+	constructor(private observationFactory: ObservationFactory) {}
+	public createCrawl(
+		network: string, //todo: configuration?
+		nodesToCrawl: NodeAddress[],
+		topTierAddresses: NodeAddress[],
+		topTierQuorumSet: QuorumSet,
+		latestConfirmedClosedLedger: Ledger,
+		quorumSets: Map<string, QuorumSet>,
+		logger: P.Logger
+	): Crawl {
+		const observation = this.observationFactory.createObservation(
+			network,
+			new Slots(topTierQuorumSet, logger),
+			topTierAddresses,
+			new PeerNodeCollection(),
+			latestConfirmedClosedLedger,
+			quorumSets
+		);
+		return new Crawl(nodesToCrawl, observation);
+	}
+}