From 419252a9082b710ad42012dd4c01cc397de8e658 Mon Sep 17 00:00:00 2001 From: Ivaylo Andonov Date: Fri, 5 Aug 2022 10:09:18 +0300 Subject: [PATCH] feat(cardano-services): cache stake pool queries - extend StakePoolProvider with cache and epochPollService - refactor sub queries and optimize caching of pools --- .../DbSyncNetworkInfoProvider.ts | 4 +- .../src/Program/loadHttpServer.ts | 10 +- .../DbSyncStakePool.ts | 149 ++++++++++++++---- .../StakePoolBuilder.ts | 20 ++- .../DbSyncStakePoolProvider/mappers.ts | 93 ++++++----- .../DbSyncStakePoolProvider/types.ts | 17 +- .../StakePool/DbSyncStakePoolProvider/util.ts | 20 +++ .../src/util/polling/EpochPollService.ts | 4 +- .../NetworkInfoHttpService.test.ts | 3 +- .../DbSyncStakePoolProvider/mappers.test.ts | 87 ++++++---- .../StakePool/StakePoolHttpService.test.ts | 18 ++- 11 files changed, 308 insertions(+), 117 deletions(-) diff --git a/packages/cardano-services/src/NetworkInfo/DbSyncNetworkInfoProvider/DbSyncNetworkInfoProvider.ts b/packages/cardano-services/src/NetworkInfo/DbSyncNetworkInfoProvider/DbSyncNetworkInfoProvider.ts index 23268787c1e..e366aa97f36 100644 --- a/packages/cardano-services/src/NetworkInfo/DbSyncNetworkInfoProvider/DbSyncNetworkInfoProvider.ts +++ b/packages/cardano-services/src/NetworkInfo/DbSyncNetworkInfoProvider/DbSyncNetworkInfoProvider.ts @@ -51,8 +51,8 @@ export class DbSyncNetworkInfoProvider extends DbSyncProvider implements Network this.#logger = logger; this.#cache = cache; this.#cardanoNode = cardanoNode; - this.#genesisDataReady = loadGenesisData(cardanoNodeConfigPath); this.#builder = new NetworkInfoBuilder(db, logger); + this.#genesisDataReady = loadGenesisData(cardanoNodeConfigPath); this.#epochPollService = EpochPollService.create(cache, () => this.#builder.queryLatestEpoch(), epochPollInterval); } @@ -119,7 +119,7 @@ export class DbSyncNetworkInfoProvider extends DbSyncProvider implements Network async close(): Promise { this.#cache.shutdown(); - this.#epochPollService.stop(); + this.#epochPollService.shutdown(); await this.#cardanoNode.shutdown(); } } diff --git a/packages/cardano-services/src/Program/loadHttpServer.ts b/packages/cardano-services/src/Program/loadHttpServer.ts index a33da5830e9..b9a85473d43 100644 --- a/packages/cardano-services/src/Program/loadHttpServer.ts +++ b/packages/cardano-services/src/Program/loadHttpServer.ts @@ -84,9 +84,13 @@ const serviceMapFactory = ( return new AssetHttpService({ assetProvider, logger }); }), - [ServiceNames.StakePool]: withDb( - (db) => new StakePoolHttpService({ logger, stakePoolProvider: new DbSyncStakePoolProvider(db, logger) }) - ), + [ServiceNames.StakePool]: withDb((db) => { + const stakePoolProvider = new DbSyncStakePoolProvider( + { epochPollInterval: args.options!.epochPollInterval! }, + { cache, db, logger } + ); + return new StakePoolHttpService({ logger, stakePoolProvider }); + }), [ServiceNames.Utxo]: withDb( (db) => new UtxoHttpService({ logger, utxoProvider: new DbSyncUtxoProvider(db, logger) }) ), diff --git a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/DbSyncStakePool.ts b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/DbSyncStakePool.ts index 03c3e167601..3e9121d3509 100644 --- a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/DbSyncStakePool.ts +++ b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/DbSyncStakePool.ts @@ -1,21 +1,45 @@ -import { CommonPoolInfo, PoolAPY, PoolData, PoolMetrics, PoolSortType } from './types'; +/* eslint-disable max-len */ +/* eslint-disable max-params */ + +import { + Cardano, + StakePoolProvider, + StakePoolQueryOptions, + StakePoolSearchResults, + StakePoolStats +} from '@cardano-sdk/core'; +import { CommonPoolInfo, OrderedResult, PoolAPY, PoolData, PoolMetrics, PoolSortType, PoolUpdate } from './types'; import { DbSyncProvider } from '../../DbSyncProvider'; +import { EpochPollService } from '../../util'; +import { IDS_NAMESPACE, StakePoolsSubQuery, getStakePoolSortType, queryCacheKey } from './util'; +import { InMemoryCache, UNLIMITED_CACHE_TTL } from '../../InMemoryCache'; import { Logger } from 'ts-log'; import { Pool } from 'pg'; import { StakePoolBuilder } from './StakePoolBuilder'; -import { StakePoolProvider, StakePoolQueryOptions, StakePoolSearchResults, StakePoolStats } from '@cardano-sdk/core'; -import { getStakePoolSortType } from './util'; import { isNotNil } from '@cardano-sdk/util'; -import { toCoreStakePool } from './mappers'; +import { toStakePoolResults } from './mappers'; + +export interface StakePoolProviderProps { + epochPollInterval: number; +} +export interface StakePoolProviderDependencies { + db: Pool; + cache: InMemoryCache; + logger: Logger; +} export class DbSyncStakePoolProvider extends DbSyncProvider implements StakePoolProvider { #builder: StakePoolBuilder; #logger: Logger; + #cache: InMemoryCache; + #epochPollService: EpochPollService; - constructor(db: Pool, logger: Logger) { + constructor({ epochPollInterval }: StakePoolProviderProps, { db, cache, logger }: StakePoolProviderDependencies) { super(db); - this.#builder = new StakePoolBuilder(db, logger); this.#logger = logger; + this.#cache = cache; + this.#builder = new StakePoolBuilder(db, logger); + this.#epochPollService = EpochPollService.create(cache, () => this.#builder.getLastEpoch(), epochPollInterval); } private getQueryBySortType( @@ -37,19 +61,16 @@ export class DbSyncStakePoolProvider extends DbSyncProvider implements StakePool } } - public async queryStakePools(options?: StakePoolQueryOptions): Promise { - const { params, query } = - options?.filters?._condition === 'or' - ? this.#builder.buildOrQuery(options?.filters) - : this.#builder.buildAndQuery(options?.filters); - this.#logger.debug('About to query pool hashes'); - const poolUpdates = await this.#builder.queryPoolHashes(query, params); + private async getPoolsDataOrdered( + poolUpdates: PoolUpdate[], + totalAdaAmount: string, + options?: StakePoolQueryOptions + ) { const hashesIds = poolUpdates.map(({ id }) => id); const updatesIds = poolUpdates.map(({ updateId }) => updateId); + this.#logger.debug(`${hashesIds.length} pools found`); - const totalAdaAmount = await this.#builder.getTotalAmountOfAda(); - this.#logger.debug('About to query stake pools by sort options'); const sortType = options?.sort?.field ? getStakePoolSortType(options.sort.field) : 'data'; const orderedResult = await this.getQueryBySortType(sortType, { hashesIds, totalAdaAmount, updatesIds })(options); const orderedResultHashIds = (orderedResult as CommonPoolInfo[]).map(({ hashId }) => hashId); @@ -78,19 +99,27 @@ export class DbSyncStakePoolProvider extends DbSyncProvider implements StakePool } else { poolDatas = orderedResult as PoolData[]; } + return { hashesIds, orderedResult, orderedResultHashIds, orderedResultUpdateIds, poolDatas, sortType }; + } + + private cacheStakePools(itemsToCache: { [hashId: number]: Cardano.StakePool }) { + for (const [hashId, pool] of Object.entries(itemsToCache)) + this.#cache.set(`${IDS_NAMESPACE}/${hashId}`, pool, UNLIMITED_CACHE_TTL); + } + private async queryExtraPoolsData( + idsToFetch: PoolUpdate[], + sortType: PoolSortType, + totalAdaAmount: string, + orderedResult: OrderedResult, + hashesIds: number[], + options?: StakePoolQueryOptions + ) { this.#logger.debug('About to query stake pool extra information'); - const [ - poolRelays, - poolOwners, - poolRegistrations, - poolRetirements, - poolRewards, - poolMetrics, - poolAPYs, - totalCount, - lastEpoch - ] = await Promise.all([ + const orderedResultHashIds = idsToFetch.map(({ id }) => id); + const orderedResultUpdateIds = idsToFetch.map(({ updateId }) => updateId); + + return await Promise.all([ // TODO: it would be easier and make the code cleaner if all queries had the same id as argument // (either hash or update id) this.#builder.queryPoolRelays(orderedResultUpdateIds), @@ -103,12 +132,57 @@ export class DbSyncStakePoolProvider extends DbSyncProvider implements StakePool : this.#builder.queryPoolMetrics(orderedResultHashIds, totalAdaAmount), sortType === 'apy' ? (orderedResult as PoolAPY[]) - : this.#builder.queryPoolAPY(hashesIds, { rewardsHistoryLimit: options?.rewardsHistoryLimit }), - this.#builder.queryTotalCount(query, params), - this.#builder.getLastEpoch() + : this.#builder.queryPoolAPY(hashesIds, { rewardsHistoryLimit: options?.rewardsHistoryLimit }) ]); + } + + public async queryStakePools(options?: StakePoolQueryOptions): Promise { + const { params, query } = + options?.filters?._condition === 'or' + ? this.#builder.buildOrQuery(options?.filters) + : this.#builder.buildAndQuery(options?.filters); - return toCoreStakePool(orderedResultHashIds, { + // Get pool updates/hashes + const poolUpdates = await this.#cache.get(queryCacheKey(StakePoolsSubQuery.POOL_HASHES, options), () => + this.#builder.queryPoolHashes(query, params) + ); + // Get pool total amount of ada + const totalAdaAmount = await this.#cache.get(queryCacheKey(StakePoolsSubQuery.TOTAL_ADA_AMOUNT), () => + this.#builder.getTotalAmountOfAda() + ); + // Get pool total stake pools count + const totalCount = await this.#builder.queryTotalCount(query, params); + // Get last epoch no + const lastEpoch = await this.#builder.getLastEpoch(); + // Get stake pools data + const { orderedResultHashIds, orderedResultUpdateIds, orderedResult, poolDatas, hashesIds, sortType } = + await this.#cache.get(queryCacheKey(StakePoolsSubQuery.POOLS_DATA_ORDERED, options), () => + this.getPoolsDataOrdered(poolUpdates, totalAdaAmount, options) + ); + + // Create lookup table with pool ids: (hashId:updateId) + const hashIdsMap = Object.fromEntries( + orderedResultHashIds.map((hashId, idx) => [hashId, orderedResultUpdateIds[idx]]) + ); + + // Create a lookup table with cached pools: (hashId:Cardano.StakePool) + const fromCache = Object.fromEntries( + orderedResultHashIds.map((hashId) => [ + hashId, + this.#cache.getVal(`${IDS_NAMESPACE}/${hashId}`) + ]) + ); + + // Compute ids to fetch from db + const idsToFetch = Object.entries(fromCache) + .filter(([_, pool]) => pool === undefined) + .map(([hashId, _]) => ({ id: Number(hashId), updateId: hashIdsMap[hashId] })); + + // Get stake pools extra information + const [poolRelays, poolOwners, poolRegistrations, poolRetirements, poolRewards, poolMetrics, poolAPYs] = + await this.queryExtraPoolsData(idsToFetch, sortType, totalAdaAmount, orderedResult, hashesIds, options); + + const { results, poolsToCache } = toStakePoolResults(orderedResultHashIds, fromCache, { lastEpoch, poolAPYs, poolDatas, @@ -120,10 +194,23 @@ export class DbSyncStakePoolProvider extends DbSyncProvider implements StakePool poolRewards: poolRewards.filter(isNotNil), totalCount }); + + // Cache stake pools core objects + this.cacheStakePools(poolsToCache); + return results; } public async stakePoolStats(): Promise { this.#logger.debug('About to query pool stats'); - return this.#builder.queryPoolStats(); + return await this.#cache.get(queryCacheKey(StakePoolsSubQuery.STATS), () => this.#builder.queryPoolStats()); + } + + async start(): Promise { + this.#epochPollService.start(); + } + + async close(): Promise { + this.#cache.shutdown(); + this.#epochPollService.shutdown(); } } diff --git a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/StakePoolBuilder.ts b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/StakePoolBuilder.ts index 3d759fdb2d8..745629ad227 100644 --- a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/StakePoolBuilder.ts +++ b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/StakePoolBuilder.ts @@ -60,11 +60,13 @@ export class StakePoolBuilder { this.#db = db; this.#logger = logger; } + public async queryRetirements(hashesIds: number[]) { this.#logger.debug('About to query pool retirements'); const result: QueryResult = await this.#db.query(Queries.findPoolsRetirements, [hashesIds]); return result.rows.length > 0 ? result.rows.map(mapPoolRetirement) : []; } + public async queryRegistrations(hashesIds: number[]) { this.#logger.debug('About to query pool registrations'); const result: QueryResult = await this.#db.query(Queries.findPoolsRegistrations, [ @@ -72,16 +74,19 @@ export class StakePoolBuilder { ]); return result.rows.length > 0 ? result.rows.map(mapPoolRegistration) : []; } + public async queryPoolRelays(updatesIds: number[]) { this.#logger.debug('About to query pool relays'); const result: QueryResult = await this.#db.query(Queries.findPoolsRelays, [updatesIds]); return result.rows.length > 0 ? result.rows.map(mapRelay) : []; } + public async queryPoolOwners(updatesIds: number[]) { this.#logger.debug('About to query pool owners'); const result: QueryResult = await this.#db.query(Queries.findPoolsOwners, [updatesIds]); return result.rows.length > 0 ? result.rows.map(mapAddressOwner) : []; } + public async queryPoolRewards(hashesIds: number[], limit?: number) { this.#logger.debug('About to query pool rewards'); return Promise.all( @@ -93,6 +98,7 @@ export class StakePoolBuilder { }) ); } + public async queryPoolAPY(hashesIds: number[], options?: StakePoolQueryOptions): Promise { this.#logger.debug('About to query pools APY'); const defaultSort: OrderByOptions[] = [{ field: 'apy', order: 'desc' }]; @@ -103,6 +109,7 @@ export class StakePoolBuilder { const result: QueryResult = await this.#db.query(queryWithSortAndPagination, [hashesIds]); return result.rows.map(mapPoolAPY); } + public async queryPoolData(updatesIds: number[], options?: StakePoolQueryOptions) { this.#logger.debug('About to query pool data'); const defaultSort: OrderByOptions[] = [ @@ -116,11 +123,14 @@ export class StakePoolBuilder { const result: QueryResult = await this.#db.query(queryWithSortAndPagination, [updatesIds]); return result.rows.length > 0 ? result.rows.map(mapPoolData) : []; } + // eslint-disable-next-line @typescript-eslint/no-explicit-any public async queryPoolHashes(query: string, params: any[] = []) { + this.#logger.debug('About to query pool hashes'); const result: QueryResult = await this.#db.query(query, params); return result.rows.length > 0 ? result.rows.map(mapPoolUpdate) : []; } + public async queryPoolMetrics(hashesIds: number[], totalAdaAmount: string, options?: StakePoolQueryOptions) { this.#logger.debug('About to query pool metrics'); const queryWithSortAndPagination = withPagination( @@ -133,6 +143,7 @@ export class StakePoolBuilder { ]); return result.rows.length > 0 ? result.rows.map(mapPoolMetrics) : []; } + public buildPoolsByIdentifierQuery( identifier: MultipleChoiceSearchFilter< Partial & Pick> @@ -148,6 +159,7 @@ export class StakePoolBuilder { `; return { id: { isPrimary: true, name: 'pools_by_identifier' }, params, query }; } + public buildPoolsByStatusQuery(status: Cardano.StakePoolStatus[]) { const whereClause = getStatusWhereClause(status); const query = ` @@ -156,6 +168,7 @@ export class StakePoolBuilder { `; return { id: { isPrimary: true, name: 'pools_by_status' }, query }; } + public buildPoolsByPledgeMetQuery(pledgeMet: boolean) { const subQueries = [...poolsByPledgeMetSubqueries]; subQueries.push({ @@ -167,16 +180,19 @@ export class StakePoolBuilder { }); return subQueries; } + public async getLastEpoch() { this.#logger.debug('About to query last epoch'); const result: QueryResult = await this.#db.query(Queries.findLastEpoch); return result.rows[0].no; } + public async getTotalAmountOfAda() { - this.#logger.debug('About to get total amount of ada'); + this.#logger.debug('About to query total ada amount'); const result: QueryResult = await this.#db.query(Queries.findTotalAda); return result.rows[0].total_ada; } + public buildOrQuery(filters: StakePoolQueryOptions['filters']) { const subQueries: SubQuery[] = []; const params = []; @@ -201,6 +217,7 @@ export class StakePoolBuilder { } return { params, query }; } + // eslint-disable-next-line max-statements public buildAndQuery(filters: StakePoolQueryOptions['filters']) { let query = Queries.findPools; @@ -257,6 +274,7 @@ export class StakePoolBuilder { query = addSentenceToQuery(query, groupByClause); return { params, query }; } + // eslint-disable-next-line @typescript-eslint/no-explicit-any public async queryTotalCount(query: string, _params: any[]) { this.#logger.debug('About to get total count of pools'); diff --git a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/mappers.ts b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/mappers.ts index f8d7fc93eb0..773abfdad51 100644 --- a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/mappers.ts +++ b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/mappers.ts @@ -1,7 +1,8 @@ -import { Cardano, StakePoolSearchResults, StakePoolStats } from '@cardano-sdk/core'; +import { Cardano, StakePoolStats } from '@cardano-sdk/core'; import { EpochReward, EpochRewardModel, + HashIdStakePoolMap, OwnerAddressModel, PoolAPY, PoolAPYModel, @@ -17,7 +18,9 @@ import { PoolRetirementModel, PoolUpdate, PoolUpdateModel, + PoolsToCache, RelayModel, + StakePoolResults, StakePoolStatsModel } from './types'; import { isNotNil } from '@cardano-sdk/util'; @@ -51,8 +54,9 @@ interface ToCoreStakePoolInput { poolAPYs: PoolAPY[]; } -export const toCoreStakePool = ( +export const toStakePoolResults = ( poolHashIds: number[], + fromCache: HashIdStakePoolMap, { poolOwners, poolDatas, @@ -65,40 +69,57 @@ export const toCoreStakePool = ( totalCount, poolAPYs }: ToCoreStakePoolInput -): StakePoolSearchResults => ({ - pageResults: poolHashIds - .map((hashId) => { - const poolData = poolDatas.find((data) => data.hashId === hashId); - if (!poolData) return; - const apy = poolAPYs.find((pool) => pool.hashId === hashId)?.apy; - const registrations = poolRegistrations.filter((r) => r.hashId === poolData.hashId); - const retirements = poolRetirements.filter((r) => r.hashId === poolData.hashId); - const metrics = poolMetrics.find((metric) => metric.hashId === poolData.hashId)?.metrics; - const toReturn: Cardano.StakePool = { - cost: poolData.cost, - epochRewards: poolRewards.filter((r) => r.hashId === poolData.hashId).map((reward) => reward.epochReward), - hexId: poolData.hexId, - id: poolData.id, - margin: poolData.margin, - metrics: metrics ? { ...metrics, apy } : ({} as Cardano.StakePoolMetrics), - owners: poolOwners.filter((o) => o.hashId === poolData.hashId).map((o) => o.address), - pledge: poolData.pledge, - relays: poolRelays.filter((r) => r.updateId === poolData.updateId).map((r) => r.relay), - rewardAccount: poolData.rewardAccount, - status: getPoolStatus(registrations[0], lastEpoch, retirements[0]), - transactions: { - registration: registrations.map((r) => r.transactionId), - retirement: retirements.map((r) => r.transactionId) - }, - vrf: poolData.vrfKeyHash - }; - if (poolData.metadata) toReturn.metadata = poolData.metadata; - if (poolData.metadataJson) toReturn.metadataJson = poolData.metadataJson; - return toReturn; - }) - .filter(isNotNil), - totalResultCount: Number(totalCount) -}); +): StakePoolResults => { + const poolsToCache: PoolsToCache = {}; + return { + poolsToCache, + results: { + pageResults: poolHashIds + .map((hashId) => { + // Get the cached value if given hash id persist in the in-memory cache + if (fromCache[hashId]) return fromCache[hashId]; + + const poolData = poolDatas.find((data) => data.hashId === hashId); + if (!poolData) return; + + const apy = poolAPYs.find((pool) => pool.hashId === hashId)?.apy; + const registrations = poolRegistrations.filter((r) => r.hashId === poolData.hashId); + const retirements = poolRetirements.filter((r) => r.hashId === poolData.hashId); + const metrics = poolMetrics.find((metric) => metric.hashId === poolData.hashId)?.metrics; + + const coreStakePool: Cardano.StakePool = { + cost: poolData.cost, + epochRewards: poolRewards + .filter((r) => r.hashId === Number(poolData.hashId)) + .map((reward) => reward.epochReward), + hexId: poolData.hexId, + id: poolData.id, + margin: poolData.margin, + metrics: metrics ? { ...metrics, apy } : ({} as Cardano.StakePoolMetrics), + owners: poolOwners.filter((o) => o.hashId === poolData.hashId).map((o) => o.address), + pledge: poolData.pledge, + relays: poolRelays.filter((r) => r.updateId === poolData.updateId).map((r) => r.relay), + rewardAccount: poolData.rewardAccount, + status: getPoolStatus(registrations[0], lastEpoch, retirements[0]), + transactions: { + registration: registrations.map((r) => r.transactionId), + retirement: retirements.map((r) => r.transactionId) + }, + vrf: poolData.vrfKeyHash + }; + if (poolData.metadata) coreStakePool.metadata = poolData.metadata; + if (poolData.metadataJson) coreStakePool.metadataJson = poolData.metadataJson; + + // Mark stake pool as pool to cache + poolsToCache[hashId] = coreStakePool; + + return coreStakePool; + }) + .filter(isNotNil), + totalResultCount: Number(totalCount) + } + }; +}; export const mapPoolUpdate = (poolUpdateModel: PoolUpdateModel): PoolUpdate => ({ id: poolUpdateModel.id, diff --git a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/types.ts b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/types.ts index f2c3fdd0ab2..9d98c15e064 100644 --- a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/types.ts +++ b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/types.ts @@ -1,4 +1,4 @@ -import { Cardano } from '@cardano-sdk/core'; +import { Cardano, StakePoolSearchResults } from '@cardano-sdk/core'; export interface PoolUpdateModel { id: number; // pool hash id update_id: number; @@ -158,3 +158,18 @@ export interface OrderByOptions { field: string; order: 'asc' | 'desc'; } + +export type THashId = number; +export type TUpdateId = number; +export type PoolIdsMap = Record; + +export type HashIdStakePoolMap = Record; + +export type OrderedResult = PoolMetrics[] | PoolData[] | PoolAPY[]; + +export type PoolsToCache = { [hashId: THashId]: Cardano.StakePool }; + +export type StakePoolResults = { + results: StakePoolSearchResults; + poolsToCache: PoolsToCache; +}; diff --git a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/util.ts b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/util.ts index ad822acc5fb..1d51f40f9c2 100644 --- a/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/util.ts +++ b/packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/util.ts @@ -13,3 +13,23 @@ export const getStakePoolSortType = (field: string): PoolSortType => { if (isPoolAPYSortField(field)) return 'apy'; throw new ProviderError(ProviderFailure.Unknown, null, 'Invalid sort field'); }; + +export const QUERIES_NAMESPACE = 'StakePoolQueries'; +export const IDS_NAMESPACE = 'StakePoolIds'; + +export enum StakePoolsSubQuery { + APY = 'apy', + STATS = 'stats', + METRICS = 'metrics', + REWARDS = 'rewards', + RELAYS = 'relays', + REGISTRATIONS = 'registrations', + OWNERS = 'owners', + RETIREMENTS = 'retirements', + TOTAL_ADA_AMOUNT = 'total_ada_amount', + POOL_HASHES = 'pool_hashes', + POOLS_DATA_ORDERED = 'pools_data_ordered' +} + +export const queryCacheKey = (queryName: StakePoolsSubQuery, ...args: unknown[]) => + `${QUERIES_NAMESPACE}/${queryName}/${JSON.stringify(args)}`; diff --git a/packages/cardano-services/src/util/polling/EpochPollService.ts b/packages/cardano-services/src/util/polling/EpochPollService.ts index f9dbfe9d300..5fead481d0c 100644 --- a/packages/cardano-services/src/util/polling/EpochPollService.ts +++ b/packages/cardano-services/src/util/polling/EpochPollService.ts @@ -56,9 +56,9 @@ export class EpochPollService { } /** - * Stops the poll execution + * Shutdown the poll execution */ - stop() { + shutdown() { clearInterval(this.#timeoutId); } } diff --git a/packages/cardano-services/test/NetworkInfo/NetworkInfoHttpService.test.ts b/packages/cardano-services/test/NetworkInfo/NetworkInfoHttpService.test.ts index 5a86903e3ce..4a7c5898664 100644 --- a/packages/cardano-services/test/NetworkInfo/NetworkInfoHttpService.test.ts +++ b/packages/cardano-services/test/NetworkInfo/NetworkInfoHttpService.test.ts @@ -219,7 +219,6 @@ describe('NetworkInfoHttpService', () => { expect(dbConnectionQuerySpy).toBeCalledTimes(stakeDbQueriesCount + DB_POLL_QUERIES_COUNT); expect(cardanoNodeStakeSpy).toHaveBeenCalledTimes(stakeNodeQueriesCount); expect(clearCacheSpy).not.toHaveBeenCalled(); - expect(dbConnectionQuerySpy).toBeCalledTimes(totalQueriesCount); }); it( @@ -304,7 +303,7 @@ describe('NetworkInfoHttpService', () => { expect(dbConnectionQuerySpy).toBeCalledTimes(dbSyncQueriesCount * 2); }); - it('should not invalidate the epoch values from the cache if there is no epoch rollover 2', async () => { + it('should not invalidate the epoch values from the cache if there is no epoch rollover', async () => { const currentEpochNo = 205; const totalDbQueriesCount = dbSyncQueriesCount + DB_POLL_QUERIES_COUNT; await provider.lovelaceSupply(); diff --git a/packages/cardano-services/test/StakePool/DbSyncStakePoolProvider/mappers.test.ts b/packages/cardano-services/test/StakePool/DbSyncStakePoolProvider/mappers.test.ts index b74d34f09bc..0e2d1a517fc 100644 --- a/packages/cardano-services/test/StakePool/DbSyncStakePoolProvider/mappers.test.ts +++ b/packages/cardano-services/test/StakePool/DbSyncStakePoolProvider/mappers.test.ts @@ -10,7 +10,7 @@ import { mapPoolStats, mapPoolUpdate, mapRelay, - toCoreStakePool + toStakePoolResults } from '../../../src'; describe('mappers', () => { @@ -198,7 +198,7 @@ describe('mappers', () => { hashId: poolAPYModel.hash_id }); }); - describe('toCoreStakePool', () => { + describe('mapAndCacheStakePools', () => { const poolOwners = [mapAddressOwner(addressOwnerModel)]; const poolDatas = [mapPoolData(poolDataModel)]; const poolRegistrations = [mapPoolRegistration(poolRegistrationModel)]; @@ -228,9 +228,11 @@ describe('mappers', () => { vrf: poolDatas[0].vrfKeyHash } as Cardano.StakePool; const totalCount = 1; - it('toCoreStakePool with retiring status', () => { + const fromCache = {}; + + it('toStakePoolResults with retiring status', () => { expect( - toCoreStakePool([poolDataModel.hash_id], { + toStakePoolResults([poolDataModel.hash_id], fromCache, { lastEpoch: poolRetirementModel.retiring_epoch - 1, poolAPYs, poolDatas, @@ -242,11 +244,16 @@ describe('mappers', () => { poolRewards, totalCount }) - ).toStrictEqual({ pageResults: [stakePool], totalResultCount: totalCount }); + ).toStrictEqual({ + poolsToCache: { [poolDataModel.hash_id]: stakePool }, + results: { pageResults: [stakePool], totalResultCount: totalCount } + }); }); - it('toCoreStakePool with retired status', () => { + it('toStakePoolResults with retired status', () => { + const stakePoolRetired = { ...stakePool, status: Cardano.StakePoolStatus.Retired }; + expect( - toCoreStakePool([poolDataModel.hash_id], { + toStakePoolResults([poolDataModel.hash_id], fromCache, { lastEpoch: poolRetirementModel.retiring_epoch + 1, poolAPYs, poolDatas, @@ -259,16 +266,28 @@ describe('mappers', () => { totalCount }) ).toStrictEqual({ - pageResults: [{ ...stakePool, status: Cardano.StakePoolStatus.Retired }], - totalResultCount: totalCount + poolsToCache: { [poolDataModel.hash_id]: stakePoolRetired }, + results: { + pageResults: [stakePoolRetired], + totalResultCount: totalCount + } }); }); - it('toCoreStakePool with activating status', () => { + it('toStakePoolResults with activating status', () => { const _retirements = [ mapPoolRetirement({ ...poolRetirementModel, retiring_epoch: poolRegistrationModel.active_epoch_no - 1 }) ]; + const stakePoolActivating = { + ...stakePool, + status: Cardano.StakePoolStatus.Activating, + transactions: { + registration: poolRegistrations.map((r) => r.transactionId), + retirement: _retirements.map((r) => r.transactionId) + } + }; + expect( - toCoreStakePool([poolDataModel.hash_id], { + toStakePoolResults([poolDataModel.hash_id], fromCache, { lastEpoch: poolRegistrationModel.active_epoch_no - 1, poolAPYs, poolDatas, @@ -281,25 +300,28 @@ describe('mappers', () => { totalCount }) ).toEqual({ - pageResults: [ - { - ...stakePool, - status: Cardano.StakePoolStatus.Activating, - transactions: { - registration: poolRegistrations.map((r) => r.transactionId), - retirement: _retirements.map((r) => r.transactionId) - } - } - ], - totalResultCount: totalCount + poolsToCache: { [poolDataModel.hash_id]: stakePoolActivating }, + results: { + pageResults: [stakePoolActivating], + totalResultCount: totalCount + } }); }); - it('toCoreStakePool with active status', () => { + it('toStakePoolResults with active status', () => { const _retirements = [ mapPoolRetirement({ ...poolRetirementModel, retiring_epoch: poolRegistrationModel.active_epoch_no }) ]; + const stakePoolActive = { + ...stakePool, + status: Cardano.StakePoolStatus.Active, + transactions: { + registration: poolRegistrations.map((r) => r.transactionId), + retirement: _retirements.map((r) => r.transactionId) + } + }; + expect( - toCoreStakePool([poolDataModel.hash_id], { + toStakePoolResults([poolDataModel.hash_id], fromCache, { lastEpoch: poolRegistrationModel.active_epoch_no, poolAPYs, poolDatas, @@ -312,20 +334,15 @@ describe('mappers', () => { totalCount }) ).toEqual({ - pageResults: [ - { - ...stakePool, - status: Cardano.StakePoolStatus.Active, - transactions: { - registration: poolRegistrations.map((r) => r.transactionId), - retirement: _retirements.map((r) => r.transactionId) - } - } - ], - totalResultCount: totalCount + poolsToCache: { [poolDataModel.hash_id]: stakePoolActive }, + results: { + pageResults: [stakePoolActive], + totalResultCount: totalCount + } }); }); }); + it('mapPoolStats', () => { expect(mapPoolStats({ active: '20', retired: '0', retiring: '1' })).toEqual({ qty: { active: 20, retired: 0, retiring: 1 } diff --git a/packages/cardano-services/test/StakePool/StakePoolHttpService.test.ts b/packages/cardano-services/test/StakePool/StakePoolHttpService.test.ts index 4b706d09f65..4e677ac99a1 100644 --- a/packages/cardano-services/test/StakePool/StakePoolHttpService.test.ts +++ b/packages/cardano-services/test/StakePool/StakePoolHttpService.test.ts @@ -14,6 +14,7 @@ import { import { CreateHttpProviderConfig, stakePoolHttpProvider } from '../../../cardano-services-client'; import { DbSyncStakePoolProvider, HttpServer, HttpServerConfig, StakePoolHttpService } from '../../src'; import { INFO, createLogger } from 'bunyan'; +import { InMemoryCache, UNLIMITED_CACHE_TTL } from '../../src/InMemoryCache'; import { Pool } from 'pg'; import { doServerRequest } from '../util'; import { getPort } from 'get-port-please'; @@ -52,7 +53,7 @@ const addPledgeMetFilter = (options: StakePoolQueryOptions, pledgeMet: boolean): }); describe('StakePoolHttpService', () => { - let dbConnection: Pool; + let db: Pool; let httpServer: HttpServer; let stakePoolProvider: DbSyncStakePoolProvider; let service: StakePoolHttpService; @@ -63,12 +64,15 @@ describe('StakePoolHttpService', () => { let doStakePoolRequest: ReturnType; let provider: StakePoolProvider; + const epochPollInterval = 2 * 1000; + const cache = new InMemoryCache(UNLIMITED_CACHE_TTL); + beforeAll(async () => { port = await getPort(); baseUrl = `http://localhost:${port}/stake-pool`; config = { listen: { port } }; clientConfig = { baseUrl, logger: createLogger({ level: INFO, name: 'unit tests' }) }; - dbConnection = new Pool({ connectionString: process.env.POSTGRES_CONNECTION_STRING }); + db = new Pool({ connectionString: process.env.POSTGRES_CONNECTION_STRING, max: 1, min: 1 }); doStakePoolRequest = doServerRequest(baseUrl); }); @@ -101,7 +105,7 @@ describe('StakePoolHttpService', () => { // eslint-disable-next-line sonarjs/cognitive-complexity describe('healthy state', () => { beforeAll(async () => { - stakePoolProvider = new DbSyncStakePoolProvider(dbConnection, logger); + stakePoolProvider = new DbSyncStakePoolProvider({ epochPollInterval }, { cache, db, logger }); service = new StakePoolHttpService({ logger, stakePoolProvider }); httpServer = new HttpServer(config, { logger, services: [service] }); await httpServer.initialize(); @@ -110,7 +114,7 @@ describe('StakePoolHttpService', () => { }); afterAll(async () => { - await dbConnection.end(); + await db.end(); await httpServer.shutdown(); }); @@ -125,6 +129,10 @@ describe('StakePoolHttpService', () => { }); describe('/search', () => { + beforeEach(async () => { + cache.clear(); + }); + const url = '/search'; describe('with Http Server', () => { it('returns a 200 coded response with a well formed HTTP request', async () => { @@ -182,6 +190,7 @@ describe('StakePoolHttpService', () => { const req = { pagination: { limit: 1, startAt: 1 } }; const reqWithRewardsPagination = { pagination: { limit: 1, startAt: 1 }, rewardsHistoryLimit: 0 }; const responseWithPagination = await provider.queryStakePools(reqWithRewardsPagination); + cache.clear(); const response = await provider.queryStakePools(req); expect(response.pageResults[0].epochRewards.length).toEqual(1); expect(responseWithPagination.pageResults[0].epochRewards.length).toEqual(0); @@ -190,6 +199,7 @@ describe('StakePoolHttpService', () => { const req: StakePoolQueryOptions = { filters: { _condition: 'or' }, pagination: { limit: 1, startAt: 1 } }; const reqWithRewardsPagination = { pagination: { limit: 1, startAt: 1 }, rewardsHistoryLimit: 0 }; const responseWithPagination = await provider.queryStakePools(reqWithRewardsPagination); + cache.clear(); const response = await provider.queryStakePools(req); expect(response.pageResults[0].epochRewards.length).toEqual(1); expect(responseWithPagination.pageResults[0].epochRewards.length).toEqual(0);