diff --git a/packages/core-api/__tests__/__support__/setup.ts b/packages/core-api/__tests__/__support__/setup.ts index 3de853d2b4..d990a0a17e 100644 --- a/packages/core-api/__tests__/__support__/setup.ts +++ b/packages/core-api/__tests__/__support__/setup.ts @@ -1,5 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; import delay from "delay"; import { registerWithContainer, setUpContainer } from "../../../core-test-utils/src/helpers/container"; import { plugin } from "../../src/plugin"; @@ -31,11 +31,11 @@ async function setUp() { ], }); - const connection = app.resolvePlugin("database"); - await connection.db.rounds.truncate(); - await connection.buildWallets(1); - await connection.saveWallets(true); - await connection.saveRound(round); + const databaseService = app.resolvePlugin("database"); + await databaseService.connection.roundsRepository.truncate(); + await databaseService.buildWallets(1); + await databaseService.saveWallets(true); + await databaseService.saveRound(round); await registerWithContainer(plugin, options); await delay(1000); // give some more time for api server to be up @@ -48,16 +48,16 @@ async function tearDown() { } async function calculateRanks() { - const connection = app.resolvePlugin("database"); + const databaseService = app.resolvePlugin("database"); - const rows = await connection.query.manyOrNone(queries.spv.delegatesRanks); + const rows = await (databaseService.connection as any).query.manyOrNone(queries.spv.delegatesRanks); rows.forEach((delegate, i) => { - const wallet = connection.walletManager.findByPublicKey(delegate.publicKey); + const wallet = databaseService.walletManager.findByPublicKey(delegate.publicKey); wallet.missedBlocks = +delegate.missedBlocks; - wallet.rate = i + 1; + (wallet as any).rate = i + 1; - connection.walletManager.reindex(wallet); + databaseService.walletManager.reindex(wallet); }); } diff --git a/packages/core-api/__tests__/v2/handlers/blocks.test.ts b/packages/core-api/__tests__/v2/handlers/blocks.test.ts index bdf574ba40..9d92841bbd 100644 --- a/packages/core-api/__tests__/v2/handlers/blocks.test.ts +++ b/packages/core-api/__tests__/v2/handlers/blocks.test.ts @@ -8,7 +8,7 @@ import { blocks2to100 } from "../../../../core-test-utils/src/fixtures"; import { resetBlockchain } from "../../../../core-test-utils/src/helpers"; import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; const container = app; const { Block } = models; @@ -146,8 +146,8 @@ describe("API 2.0 - Blocks", () => { it("should POST a search for blocks with the exact specified previousBlock", async () => { // save a new block so that we can make the request with previousBlock const block2 = new Block(blocks2to100[0]); - const database = container.resolvePlugin("database"); - await database.saveBlock(block2); + const databaseService = container.resolvePlugin("database"); + await databaseService.saveBlock(block2); const response = await utils[request]("POST", "blocks/search", { id: blocks2to100[0].id, @@ -163,7 +163,7 @@ describe("API 2.0 - Blocks", () => { expect(block.id).toBe(blocks2to100[0].id); expect(block.previous).toBe(blocks2to100[0].previousBlock); - await database.deleteBlock(block2); // reset to genesis block + await databaseService.deleteBlock(block2); // reset to genesis block }); }, ); diff --git a/packages/core-api/__tests__/v2/handlers/delegates.test.ts b/packages/core-api/__tests__/v2/handlers/delegates.test.ts index ae129a3aa8..e4a9882365 100644 --- a/packages/core-api/__tests__/v2/handlers/delegates.test.ts +++ b/packages/core-api/__tests__/v2/handlers/delegates.test.ts @@ -8,7 +8,7 @@ import { models } from "@arkecosystem/crypto"; const { Block } = models; import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; const delegate = { username: "genesis_9", @@ -155,8 +155,8 @@ describe("API 2.0 - Delegates", () => { it("should GET all blocks for a delegate by the given identifier", async () => { // save a new block so that we can make the request with generatorPublicKey const block2 = new Block(blocks2to100[0]); - const database = app.resolvePlugin("database"); - await database.saveBlock(block2); + const databaseService = app.resolvePlugin("database"); + await databaseService.saveBlock(block2); const response = await utils[request]( "GET", @@ -166,7 +166,7 @@ describe("API 2.0 - Delegates", () => { expect(response.data.data).toBeArray(); utils.expectBlock(response.data.data[0]); - await database.deleteBlock(block2); // reset to genesis block + await databaseService.deleteBlock(block2); // reset to genesis block }); }, ); diff --git a/packages/core-api/package.json b/packages/core-api/package.json index 5ca7ef6240..420bd0fbc1 100644 --- a/packages/core-api/package.json +++ b/packages/core-api/package.json @@ -31,7 +31,6 @@ "dependencies": { "@arkecosystem/core-interfaces": "^2.1.0", "@arkecosystem/core-container": "^2.1.0", - "@arkecosystem/core-database-postgres": "^2.1.0", "@arkecosystem/core-http-utils": "^2.1.0", "@arkecosystem/core-transaction-pool": "^2.1.0", "@arkecosystem/core-utils": "^2.1.0", diff --git a/packages/core-api/src/interfaces/repository.ts b/packages/core-api/src/interfaces/repository.ts index b068e3100c..f91ed6026c 100644 --- a/packages/core-api/src/interfaces/repository.ts +++ b/packages/core-api/src/interfaces/repository.ts @@ -1,5 +1,5 @@ export interface IRepository { - database: any; + databaseService: any; cache: any; model: any; query: any; diff --git a/packages/core-api/src/repositories/blocks.ts b/packages/core-api/src/repositories/blocks.ts index 61a8e95c64..9d312fbf50 100644 --- a/packages/core-api/src/repositories/blocks.ts +++ b/packages/core-api/src/repositories/blocks.ts @@ -119,7 +119,7 @@ export class BlockRepository extends Repository implements IRepository { } public getModel(): any { - return this.database.models.block; + return (this.databaseService.connection as any).models.block; } public __orderBy(parameters): string[] { diff --git a/packages/core-api/src/repositories/repository.ts b/packages/core-api/src/repositories/repository.ts index 5d2738f805..5b4499a32b 100644 --- a/packages/core-api/src/repositories/repository.ts +++ b/packages/core-api/src/repositories/repository.ts @@ -1,12 +1,11 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { TransactionPool } from "@arkecosystem/core-interfaces"; +import { Database, TransactionPool } from "@arkecosystem/core-interfaces"; import snakeCase from "lodash/snakeCase"; import { IRepository } from "../interfaces"; export abstract class Repository implements IRepository { - public database = app.resolvePlugin("database"); - public cache = this.database.getCache(); + public databaseService = app.resolvePlugin("database"); + public cache = this.databaseService.cache; public transactionPool = app.resolvePlugin("transactionPool"); public model = this.getModel(); public query = this.model.query(); @@ -20,11 +19,11 @@ export abstract class Repository implements IRepository { public abstract getModel(): any; public async _find(query): Promise { - return this.database.query.oneOrNone(query.toQuery()); + return (this.databaseService.connection as any).query.oneOrNone(query.toQuery()); } public async _findMany(query): Promise { - return this.database.query.manyOrNone(query.toQuery()); + return (this.databaseService.connection as any).query.manyOrNone(query.toQuery()); } public async _findManyWithCount(selectQuery, { limit, offset, orderBy }): Promise { @@ -61,7 +60,7 @@ export abstract class Repository implements IRepository { let count = 0; const explainSql = `EXPLAIN ${selectQuery.toString()}`; - for (const row of await this.database.query.manyOrNone(explainSql)) { + for (const row of await (this.databaseService.connection as any).query.manyOrNone(explainSql)) { const line: any = Object.values(row)[0]; const match = line.match(/rows=([0-9]+)/); if (match !== null) { diff --git a/packages/core-api/src/repositories/transactions.ts b/packages/core-api/src/repositories/transactions.ts index 338434636b..280e5686e7 100644 --- a/packages/core-api/src/repositories/transactions.ts +++ b/packages/core-api/src/repositories/transactions.ts @@ -41,7 +41,7 @@ export class TransactionsRepository extends Repository implements IRepository { } if (parameters.ownerId) { - const owner = this.database.walletManager.findByAddress(parameters.ownerId); + const owner = this.databaseService.walletManager.findByAddress(parameters.ownerId); selectQuery.and(this.query.sender_public_key.equals(owner.publicKey)); selectQuery.or(this.query.recipient_id.equals(owner.address)); @@ -394,7 +394,7 @@ export class TransactionsRepository extends Repository implements IRepository { } public getModel(): object { - return this.database.models.transaction; + return (this.databaseService.connection as any).models.transaction; } /** @@ -403,7 +403,7 @@ export class TransactionsRepository extends Repository implements IRepository { * @return {Object} */ public async __mapBlocksToTransactions(data): Promise { - const blockQuery = this.database.models.block.query(); + const blockQuery = (this.databaseService.connection as any).models.block.query(); // Array... if (Array.isArray(data)) { @@ -493,8 +493,8 @@ export class TransactionsRepository extends Repository implements IRepository { * @return {String} */ public __publicKeyFromAddress(senderId): string { - if (this.database.walletManager.exists(senderId)) { - return this.database.walletManager.findByAddress(senderId).publicKey; + if (this.databaseService.walletManager.exists(senderId)) { + return this.databaseService.walletManager.findByAddress(senderId).publicKey; } return null; diff --git a/packages/core-api/src/versions/1/accounts/controller.ts b/packages/core-api/src/versions/1/accounts/controller.ts index b57dfd11fb..5c8a6410cc 100644 --- a/packages/core-api/src/versions/1/accounts/controller.ts +++ b/packages/core-api/src/versions/1/accounts/controller.ts @@ -60,7 +60,7 @@ export class AccountsController extends Controller { public async delegates(request: Hapi.Request, h: Hapi.ResponseToolkit) { try { // @ts-ignore - const account = await this.database.wallets.findById(request.query.address); + const account = await this.databaseService.wallets.findById(request.query.address); if (!account) { return super.respondWith("Address not found.", true); @@ -74,7 +74,7 @@ export class AccountsController extends Controller { ); } - const delegate = await this.database.delegates.findById(account.vote); + const delegate = await this.databaseService.delegates.findById(account.vote); return super.respondWith({ delegates: [super.toResource(request, delegate, "delegate")], @@ -86,9 +86,9 @@ export class AccountsController extends Controller { public async top(request: Hapi.Request, h: Hapi.ResponseToolkit) { try { - let accounts = this.database.wallets.top(super.paginate(request)); + const wallets = this.databaseService.wallets.top(super.paginate(request)); - accounts = accounts.rows.map(account => ({ + const accounts = wallets.rows.map(account => ({ address: account.address, balance: `${account.balance}`, publicKey: account.publicKey, @@ -102,7 +102,7 @@ export class AccountsController extends Controller { public async count(request: Hapi.Request, h: Hapi.ResponseToolkit) { try { - const { count } = await this.database.wallets.findAll(); + const { count } = await this.databaseService.wallets.findAll(); return super.respondWith({ count }); } catch (error) { diff --git a/packages/core-api/src/versions/1/accounts/methods.ts b/packages/core-api/src/versions/1/accounts/methods.ts index 04b1794c35..ce6e32fa63 100644 --- a/packages/core-api/src/versions/1/accounts/methods.ts +++ b/packages/core-api/src/versions/1/accounts/methods.ts @@ -1,12 +1,12 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; import { ServerCache } from "../../../services"; import { paginate, respondWith, toCollection, toResource } from "../utils"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); const index = async request => { - const { rows } = await database.wallets.findAll({ + const { rows } = await databaseService.wallets.findAll({ ...request.query, ...paginate(request), }); @@ -17,7 +17,7 @@ const index = async request => { }; const show = async request => { - const account = await database.wallets.findById(request.query.address); + const account = await databaseService.wallets.findById(request.query.address); if (!account) { return respondWith("Account not found", true); @@ -29,7 +29,7 @@ const show = async request => { }; const balance = async request => { - const account = await database.wallets.findById(request.query.address); + const account = await databaseService.wallets.findById(request.query.address); if (!account) { return respondWith({ balance: "0", unconfirmedBalance: "0" }); @@ -42,7 +42,7 @@ const balance = async request => { }; const publicKey = async request => { - const account = await database.wallets.findById(request.query.address); + const account = await databaseService.wallets.findById(request.query.address); if (!account) { return respondWith("Account not found", true); diff --git a/packages/core-api/src/versions/1/delegates/controller.ts b/packages/core-api/src/versions/1/delegates/controller.ts index c16dd5996a..4ee2ce26f1 100644 --- a/packages/core-api/src/versions/1/delegates/controller.ts +++ b/packages/core-api/src/versions/1/delegates/controller.ts @@ -71,7 +71,7 @@ export class DelegatesController extends Controller { public async forged(request: Hapi.Request, h: Hapi.ResponseToolkit) { try { - const wallet = this.database.walletManager.findByPublicKey( + const wallet = this.databaseService.walletManager.findByPublicKey( // @ts-ignore request.query.generatorPublicKey, ); @@ -95,7 +95,7 @@ export class DelegatesController extends Controller { const delegatesCount = this.config.getMilestone(lastBlock).activeDelegates; const currentSlot = slots.getSlotNumber(lastBlock.data.timestamp); - let activeDelegates = await this.database.getActiveDelegates(lastBlock.data.height); + let activeDelegates = await this.databaseService.getActiveDelegates(lastBlock.data.height); activeDelegates = activeDelegates.map(delegate => delegate.publicKey); const nextForgers = []; diff --git a/packages/core-api/src/versions/1/delegates/methods.ts b/packages/core-api/src/versions/1/delegates/methods.ts index c5b7ed2a2d..a0024fd189 100644 --- a/packages/core-api/src/versions/1/delegates/methods.ts +++ b/packages/core-api/src/versions/1/delegates/methods.ts @@ -1,12 +1,12 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; import { ServerCache } from "../../../services"; import { paginate, respondWith, toCollection, toResource } from "../utils"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); const index = async request => { - const { count, rows } = await database.delegates.paginate({ + const { count, rows } = await databaseService.delegates.findAll({ ...request.query, ...{ offset: request.query.offset || 0, @@ -25,7 +25,7 @@ const show = async request => { return respondWith("Delegate not found", true); } - const delegate = await database.delegates.findById(request.query.publicKey || request.query.username); + const delegate = await databaseService.delegates.findById(request.query.publicKey || request.query.username); if (!delegate) { return respondWith("Delegate not found", true); @@ -37,13 +37,13 @@ const show = async request => { }; const countDelegates = async request => { - const delegate = await database.delegates.findAll(); + const delegate = await databaseService.delegates.findAll(); return respondWith({ count: delegate.count }); }; const search = async request => { - const { rows } = await database.delegates.search({ + const { rows } = await databaseService.delegates.search({ ...{ username: request.query.q }, ...paginate(request), }); @@ -54,7 +54,7 @@ const search = async request => { }; const voters = async request => { - const delegate = await database.delegates.findById(request.query.publicKey); + const delegate = await databaseService.delegates.findById(request.query.publicKey); if (!delegate) { return respondWith({ @@ -62,7 +62,7 @@ const voters = async request => { }); } - const accounts = await database.wallets.findAllByVote(delegate.publicKey); + const accounts = await databaseService.wallets.findAllByVote(delegate.publicKey); return respondWith({ accounts: toCollection(request, accounts.rows, "voter"), diff --git a/packages/core-api/src/versions/1/shared/controller.ts b/packages/core-api/src/versions/1/shared/controller.ts index ca3f735673..7e43933440 100644 --- a/packages/core-api/src/versions/1/shared/controller.ts +++ b/packages/core-api/src/versions/1/shared/controller.ts @@ -1,13 +1,12 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain, Logger } from "@arkecosystem/core-interfaces"; +import { Blockchain, Database, Logger } from "@arkecosystem/core-interfaces"; import Hapi from "hapi"; import { paginate, respondWith, respondWithCache, toCollection, toResource } from "../utils"; export class Controller { protected config = app.getConfig(); protected blockchain = app.resolvePlugin("blockchain"); - protected database = app.resolvePlugin("database"); + protected databaseService = app.resolvePlugin("database"); protected logger = app.resolvePlugin("logger"); protected paginate(request: Hapi.Request): any { diff --git a/packages/core-api/src/versions/2/blocks/transformer.ts b/packages/core-api/src/versions/2/blocks/transformer.ts index ab2780245c..cd22eb234e 100644 --- a/packages/core-api/src/versions/2/blocks/transformer.ts +++ b/packages/core-api/src/versions/2/blocks/transformer.ts @@ -1,10 +1,10 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; import { bignumify, formatTimestamp } from "@arkecosystem/core-utils"; export function transformBlock(model) { - const database = app.resolvePlugin("database"); - const generator = database.walletManager.findByPublicKey(model.generatorPublicKey); + const databaseService = app.resolvePlugin("database"); + const generator = databaseService.walletManager.findByPublicKey(model.generatorPublicKey); model.reward = bignumify(model.reward); model.totalFee = bignumify(model.totalFee); diff --git a/packages/core-api/src/versions/2/delegates/methods.ts b/packages/core-api/src/versions/2/delegates/methods.ts index fcf2691f20..be2342af89 100644 --- a/packages/core-api/src/versions/2/delegates/methods.ts +++ b/packages/core-api/src/versions/2/delegates/methods.ts @@ -1,15 +1,15 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; import Boom from "boom"; import orderBy from "lodash/orderBy"; import { blocksRepository } from "../../../repositories"; import { ServerCache } from "../../../services"; import { paginate, respondWithResource, toPagination } from "../utils"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); const index = async request => { - const delegates = await database.delegates.paginate({ + const delegates = await databaseService.delegates.findAll({ ...request.query, ...paginate(request), }); @@ -18,7 +18,7 @@ const index = async request => { }; const show = async request => { - const delegate = await database.delegates.findById(request.params.id); + const delegate = await databaseService.delegates.findById(request.params.id); if (!delegate) { return Boom.notFound("Delegate not found"); @@ -28,7 +28,7 @@ const show = async request => { }; const search = async request => { - const delegates = await database.delegates.search({ + const delegates = await databaseService.delegates.search({ ...request.payload, ...request.query, ...paginate(request), @@ -38,7 +38,7 @@ const search = async request => { }; const blocks = async request => { - const delegate = await database.delegates.findById(request.params.id); + const delegate = await databaseService.delegates.findById(request.params.id); if (!delegate) { return Boom.notFound("Delegate not found"); @@ -50,25 +50,25 @@ const blocks = async request => { }; const voters = async request => { - const delegate = await database.delegates.findById(request.params.id); + const delegate = await databaseService.delegates.findById(request.params.id); if (!delegate) { return Boom.notFound("Delegate not found"); } - const wallets = await database.wallets.findAllByVote(delegate.publicKey, paginate(request)); + const wallets = await databaseService.wallets.findAllByVote(delegate.publicKey, paginate(request)); return toPagination(request, wallets, "wallet"); }; const voterBalances = async request => { - const delegate = await database.delegates.findById(request.params.id); + const delegate = await databaseService.delegates.findById(request.params.id); if (!delegate) { return Boom.notFound("Delegate not found"); } - const wallets = await database.wallets.all().filter(wallet => wallet.vote === delegate.publicKey); + const wallets = await databaseService.wallets.all().filter(wallet => wallet.vote === delegate.publicKey); const data = {}; orderBy(wallets, ["balance"], ["desc"]).forEach(wallet => { diff --git a/packages/core-api/src/versions/2/shared/controller.ts b/packages/core-api/src/versions/2/shared/controller.ts index a6df0c0dc6..659a8474d2 100644 --- a/packages/core-api/src/versions/2/shared/controller.ts +++ b/packages/core-api/src/versions/2/shared/controller.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain } from "@arkecosystem/core-interfaces"; +import { Blockchain, Database } from "@arkecosystem/core-interfaces"; import Hapi from "hapi"; import { paginate, @@ -15,7 +14,7 @@ import { export class Controller { protected config = app.getConfig(); protected blockchain = app.resolvePlugin("blockchain"); - protected database = app.resolvePlugin("database"); + protected databaseService = app.resolvePlugin("database"); protected paginate(request: Hapi.Request): any { return paginate(request); diff --git a/packages/core-api/src/versions/2/wallets/methods.ts b/packages/core-api/src/versions/2/wallets/methods.ts index 49929bf8f5..c354ff87f5 100644 --- a/packages/core-api/src/versions/2/wallets/methods.ts +++ b/packages/core-api/src/versions/2/wallets/methods.ts @@ -1,14 +1,14 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; import Boom from "boom"; import { transactionsRepository } from "../../../repositories"; import { ServerCache } from "../../../services"; import { paginate, respondWithResource, toPagination } from "../utils"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); const index = async request => { - const wallets = await database.wallets.findAll({ + const wallets = await databaseService.wallets.findAll({ ...request.query, ...paginate(request), }); @@ -17,13 +17,13 @@ const index = async request => { }; const top = async request => { - const wallets = await database.wallets.top(paginate(request)); + const wallets = await databaseService.wallets.top(paginate(request)); return toPagination(request, wallets, "wallet"); }; const show = async request => { - const wallet = await database.wallets.findById(request.params.id); + const wallet = await databaseService.wallets.findById(request.params.id); if (!wallet) { return Boom.notFound("Wallet not found"); @@ -33,7 +33,7 @@ const show = async request => { }; const transactions = async request => { - const wallet = await database.wallets.findById(request.params.id); + const wallet = await databaseService.wallets.findById(request.params.id); if (!wallet) { return Boom.notFound("Wallet not found"); @@ -49,7 +49,7 @@ const transactions = async request => { }; const transactionsSent = async request => { - const wallet = await database.wallets.findById(request.params.id); + const wallet = await databaseService.wallets.findById(request.params.id); if (!wallet) { return Boom.notFound("Wallet not found"); @@ -68,7 +68,7 @@ const transactionsSent = async request => { }; const transactionsReceived = async request => { - const wallet = await database.wallets.findById(request.params.id); + const wallet = await databaseService.wallets.findById(request.params.id); if (!wallet) { return Boom.notFound("Wallet not found"); @@ -87,7 +87,7 @@ const transactionsReceived = async request => { }; const votes = async request => { - const wallet = await database.wallets.findById(request.params.id); + const wallet = await databaseService.wallets.findById(request.params.id); if (!wallet) { return Boom.notFound("Wallet not found"); @@ -105,7 +105,7 @@ const votes = async request => { }; const search = async request => { - const wallets = await database.wallets.search({ + const wallets = await databaseService.wallets.search({ ...request.payload, ...request.query, ...paginate(request), diff --git a/packages/core-blockchain/package.json b/packages/core-blockchain/package.json index ac6e775b4b..ed89405356 100644 --- a/packages/core-blockchain/package.json +++ b/packages/core-blockchain/package.json @@ -32,7 +32,6 @@ "dependencies": { "@arkecosystem/core-interfaces": "^2.1.0", "@arkecosystem/core-container": "^2.1.0", - "@arkecosystem/core-database-postgres": "^2.1.0", "@arkecosystem/core-utils": "^2.1.0", "@arkecosystem/crypto": "^2.1.0", "@types/lodash.get": "^4.4.4", diff --git a/packages/core-blockchain/src/blockchain.ts b/packages/core-blockchain/src/blockchain.ts index f1d10cc387..9e2dea2f8e 100644 --- a/packages/core-blockchain/src/blockchain.ts +++ b/packages/core-blockchain/src/blockchain.ts @@ -1,7 +1,13 @@ /* tslint:disable:max-line-length */ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain as blockchain, EventEmitter, Logger, P2P, TransactionPool } from "@arkecosystem/core-interfaces"; +import { + Blockchain as blockchain, + Database, + EventEmitter, + Logger, + P2P, + TransactionPool, +} from "@arkecosystem/core-interfaces"; import { models, slots } from "@arkecosystem/crypto"; import delay from "delay"; @@ -47,7 +53,7 @@ export class Blockchain implements blockchain.IBlockchain { * @return {ConnectionInterface} */ get database() { - return app.resolvePlugin("database"); + return app.resolvePlugin("database"); } public isStopped: boolean; @@ -377,7 +383,11 @@ export class Blockchain implements blockchain.IBlockchain { const blocks = await this.database.getTopBlocks(count); logger.info( - `Removing ${pluralize("block", blocks.length, true)} from height ${blocks[0].height.toLocaleString()}`, + `Removing ${pluralize( + "block", + blocks.length, + true, + )} from height ${(blocks[0] as any).height.toLocaleString()}`, ); for (let block of blocks) { diff --git a/packages/core-container/src/container.ts b/packages/core-container/src/container.ts index 378c4c3323..0856862dda 100644 --- a/packages/core-container/src/container.ts +++ b/packages/core-container/src/container.ts @@ -113,6 +113,7 @@ export class Container implements container.IContainer { * @throws {Error} */ public resolve(key): T { + try { return this.container.resolve(key); } catch (err) { diff --git a/packages/core-database-postgres/__tests__/connection.test.ts b/packages/core-database-postgres/__tests__/connection.test.ts index bb3ba721ed..86ded19431 100644 --- a/packages/core-database-postgres/__tests__/connection.test.ts +++ b/packages/core-database-postgres/__tests__/connection.test.ts @@ -1,16 +1,16 @@ import { app } from "@arkecosystem/core-container"; +import { Database } from "@arkecosystem/core-interfaces"; import { models } from "@arkecosystem/crypto"; import genesisBlock from "../../core-test-utils/src/config/testnet/genesisBlock.json"; -import { PostgresConnection } from "../src/connection"; import { setUp, tearDown } from "./__support__/setup"; const { Block } = models; -let connection; +let databaseService: Database.IDatabaseService; beforeAll(async () => { await setUp(); - connection = app.resolvePlugin("database"); + databaseService = app.resolvePlugin("database"); }); afterAll(async () => { @@ -20,7 +20,7 @@ afterAll(async () => { describe("Connection", () => { describe("verifyBlockchain", () => { it("should be valid - no errors - when verifying blockchain", async () => { - expect(await connection.verifyBlockchain()).toEqual({ + expect(await databaseService.verifyBlockchain()).toEqual({ valid: true, errors: [], }); @@ -29,7 +29,7 @@ describe("Connection", () => { describe("getLastBlock", () => { it("should get the genesis block as last block", async () => { - const lastBlock = await connection.getLastBlock(); + const lastBlock = await databaseService.getLastBlock(); expect(lastBlock).toEqual(new Block(genesisBlock)); }); diff --git a/packages/core-database-postgres/src/connection.ts b/packages/core-database-postgres/src/connection.ts deleted file mode 100644 index da3c8e27e2..0000000000 --- a/packages/core-database-postgres/src/connection.ts +++ /dev/null @@ -1,704 +0,0 @@ -import crypto from "crypto"; -import fs from "fs"; -import chunk from "lodash/chunk"; -import path from "path"; -import pgPromise from "pg-promise"; -import pluralize from "pluralize"; - -import { ConnectionInterface } from "@arkecosystem/core-database"; - -import { app } from "@arkecosystem/core-container"; - -import { roundCalculator } from "@arkecosystem/core-utils"; -import { Bignum, models } from "@arkecosystem/crypto"; - -import { SPV } from "./spv"; - -import { migrations } from "./migrations"; -import { Model } from "./models"; -import { repositories } from "./repositories"; -import { QueryExecutor } from "./sql/query-executor"; -import { camelizeColumns } from "./utils"; - -const { Block, Transaction } = models; - -export class PostgresConnection extends ConnectionInterface { - public models: { [key: string]: Model } = {}; - public query: QueryExecutor; - public db: any; - private cache: Map; - private pgp: any; - private spvFinished: boolean; - - public constructor(readonly options: any) { - super(options); - } - - /** - * Make the database connection instance. - * @return {PostgresConnection} - */ - public async make() { - if (this.db) { - throw new Error("Database connection already initialised"); - } - - this.logger.debug("Connecting to database"); - - this.queuedQueries = null; - this.cache = new Map(); - - try { - await this.connect(); - await this.__registerQueryExecutor(); - await this.__runMigrations(); - await this.__registerModels(); - await super._registerRepositories(); - await super._registerWalletManager(); - await this.loadBlocksFromCurrentRound(); - this.logger.debug("Connected to database."); - - return this; - } catch (error) { - app.forceExit("Unable to connect to the database!", error); - } - - return null; - } - - /** - * Connect to the database. - * @return {void} - */ - public async connect() { - const initialization = { - receive(data, result, e) { - camelizeColumns(pgp, data); - }, - extend(object) { - for (const repository of Object.keys(repositories)) { - object[repository] = new repositories[repository](object, pgp); - } - }, - }; - - const pgp = pgPromise({ ...this.options.initialization, ...initialization }); - - this.pgp = pgp; - this.db = this.pgp(this.options.connection); - } - - /** - * Disconnects from the database and closes the cache. - * @return {Promise} The successfulness of closing the Sequelize connection - */ - public async disconnect() { - try { - await this.commitQueuedQueries(); - this.cache.clear(); - } catch (error) { - this.logger.warn("Issue in commiting blocks, database might be corrupted"); - this.logger.warn(error.message); - } - - this.logger.debug("Disconnecting from database"); - - return this.pgp.end(); - } - - /** - * Verify the blockchain stored on db is not corrupted making simple assertions: - * - Last block is available - * - Last block height equals the number of stored blocks - * - Number of stored transactions equals the sum of block.numberOfTransactions in the database - * - Sum of all tx fees equals the sum of block.totalFee - * - Sum of all tx amount equals the sum of block.totalAmount - * @return {Object} An object { valid, errors } with the result of the verification and the errors - */ - public async verifyBlockchain() { - const errors = []; - - const lastBlock = await this.getLastBlock(); - - // Last block is available - if (!lastBlock) { - errors.push("Last block is not available"); - } else { - const { count: numberOfBlocks } = await this.db.blocks.count(); - - // Last block height equals the number of stored blocks - if (lastBlock.data.height !== +numberOfBlocks) { - errors.push( - `Last block height: ${lastBlock.data.height.toLocaleString()}, number of stored blocks: ${numberOfBlocks}`, - ); - } - } - - const blockStats = await this.db.blocks.statistics(); - const transactionStats = await this.db.transactions.statistics(); - - // Number of stored transactions equals the sum of block.numberOfTransactions in the database - if (blockStats.numberOfTransactions !== transactionStats.count) { - errors.push( - `Number of transactions: ${transactionStats.count}, number of transactions included in blocks: ${ - blockStats.numberOfTransactions - }`, - ); - } - - // Sum of all tx fees equals the sum of block.totalFee - if (blockStats.totalFee !== transactionStats.totalFee) { - errors.push( - `Total transaction fees: ${transactionStats.totalFee}, total of block.totalFee : ${ - blockStats.totalFee - }`, - ); - } - - // Sum of all tx amount equals the sum of block.totalAmount - if (blockStats.totalAmount !== transactionStats.totalAmount) { - errors.push( - `Total transaction amounts: ${transactionStats.totalAmount}, total of block.totalAmount : ${ - blockStats.totalAmount - }`, - ); - } - - return { - valid: !errors.length, - errors, - }; - } - - /** - * Get the top 51 delegates. - * @param {Number} height - * @param {Array} delegates - * @return {Array} - */ - public async getActiveDelegates(height, delegates?) { - const maxDelegates = this.config.getMilestone(height).activeDelegates; - const round = Math.floor((height - 1) / maxDelegates) + 1; - - if (this.forgingDelegates && this.forgingDelegates.length && this.forgingDelegates[0].round === round) { - return this.forgingDelegates; - } - - // When called during applyRound we already know the delegates, so we don't have to query the database. - if (!delegates || delegates.length === 0) { - delegates = await this.db.rounds.findById(round); - } - - const seedSource = round.toString(); - let currentSeed = crypto - .createHash("sha256") - .update(seedSource, "utf8") - .digest(); - - for (let i = 0, delCount = delegates.length; i < delCount; i++) { - for (let x = 0; x < 4 && i < delCount; i++, x++) { - const newIndex = currentSeed[x] % delCount; - const b = delegates[newIndex]; - delegates[newIndex] = delegates[i]; - delegates[i] = b; - } - currentSeed = crypto - .createHash("sha256") - .update(currentSeed) - .digest(); - } - - this.forgingDelegates = delegates.map(delegate => { - delegate.round = +delegate.round; - return delegate; - }); - - return this.forgingDelegates; - } - - /** - * Store the given round. - * @param {Array} delegates - * @return {Array} - */ - public async saveRound(delegates) { - this.logger.info(`Saving round ${delegates[0].round.toLocaleString()}`); - - await this.db.rounds.create(delegates); - - this.emitter.emit("round.created", delegates); - } - - /** - * Delete the given round. - * @param {Number} round - * @return {Promise} - */ - public async deleteRound(round) { - return this.db.rounds.delete(round); - } - - /** - * Load a list of wallets into memory. - * @param {Number} height - * @return {Boolean} success - */ - public async buildWallets(height) { - this.walletManager.reset(); - - const spvPath = `${process.env.CORE_PATH_CACHE}/spv.json`; - - if (fs.existsSync(spvPath)) { - (fs as any).removeSync(spvPath); - - this.logger.info("Ark Core ended unexpectedly - resuming from where we left off :runner:"); - - return true; - } - - try { - const spv = new SPV(this); - const success = await spv.build(height); - - this.spvFinished = true; - - await this.__registerListeners(); - - return success; - } catch (error) { - this.logger.error(error.stack); - } - - return false; - } - - /** - * Load all wallets from database. - * @return {Array} - */ - public async loadWallets() { - const wallets = await this.db.wallets.all(); - - this.walletManager.index(wallets); - - return this.walletManager.all(); - } - - /** - * Commit wallets from the memory. - * @param {Boolean} force - * @return {void} - */ - public async saveWallets(force) { - const wallets = this.walletManager - .allByPublicKey() - .filter(wallet => wallet.publicKey && (force || wallet.dirty)); - - // Remove dirty flags first to not save all dirty wallets in the exit handler - // when called during a force insert right after SPV. - this.walletManager.clear(); - - if (force) { - // all wallets to be updated, performance is better without upsert - await this.db.wallets.truncate(); - - try { - const chunks = chunk(wallets, 5000).map(c => this.db.wallets.create(c)); - await this.db.tx(t => t.batch(chunks)); - } catch (error) { - this.logger.error(error.stack); - } - } else { - // NOTE: The list of delegates is calculated in-memory against the WalletManager, - // so it is safe to perform the costly UPSERT non-blocking during round change only: - // 'await saveWallets(false)' -> 'saveWallets(false)' - try { - const queries = wallets.map(wallet => this.db.wallets.updateOrCreate(wallet)); - await this.db.tx(t => t.batch(queries)); - } catch (error) { - this.logger.error(error.stack); - } - } - - this.logger.info(`${wallets.length} modified ${pluralize("wallet", wallets.length)} committed to database`); - - this.emitter.emit("wallet.saved", wallets.length); - - // NOTE: commented out as more use cases to be taken care of - // this.walletManager.purgeEmptyNonDelegates() - } - - /** - * Commit the given block. - * NOTE: to be used when node is in sync and committing newly received blocks - * @param {Block} block - * @return {void} - */ - public async saveBlock(block) { - try { - const queries = [this.db.blocks.create(block.data)]; - - if (block.transactions.length > 0) { - queries.push(this.db.transactions.create(block.transactions)); - } - - await this.db.tx(t => t.batch(queries)); - } catch (err) { - this.logger.error(err.message); - } - } - - /** - * Delete the given block. - * @param {Block} block - * @return {void} - */ - public async deleteBlock(block) { - try { - const queries = [this.db.transactions.deleteByBlock(block.data.id), this.db.blocks.delete(block.data.id)]; - - await this.db.tx(t => t.batch(queries)); - } catch (error) { - this.logger.error(error.stack); - - throw error; - } - } - - /** - * Stores the block in memory. Generated insert statements are stored in - * `this.queuedQueries`, to be later saved to the database by calling commit. - * NOTE: to use when rebuilding to decrease the number of database tx, and - * commit blocks (save only every 1000s for instance) by calling commit. - * @param {Block} block - * @return {void} - */ - public enqueueSaveBlock(block) { - const queries = [this.db.blocks.create(block.data)]; - - if (block.transactions.length > 0) { - queries.push(this.db.transactions.create(block.transactions)); - } - - this.enqueueQueries(queries); - } - - /** - * Generated delete statements are stored in this.queuedQueries to be later - * executed by calling this.commitQueuedQueries. - * See also enqueueSaveBlock. - * @param {Block} block - * @return {void} - */ - public enqueueDeleteBlock(block) { - const queries = [this.db.transactions.deleteByBlock(block.data.id), this.db.blocks.delete(block.data.id)]; - - this.enqueueQueries(queries); - } - - /** - * Generated delete statements are stored in this.queuedQueries to be later - * executed by calling this.commitQueuedQueries. - * @param {Number} round - * @return {void} - */ - public enqueueDeleteRound(height) { - const { round, nextRound, maxDelegates } = roundCalculator.calculateRound(height); - - if (nextRound === round + 1 && height >= maxDelegates) { - this.enqueueQueries([this.db.rounds.delete(nextRound)]); - } - } - - /** - * Add queries to the queue to be executed when calling commit. - * @param {Array} queries - */ - public enqueueQueries(queries) { - if (!this.queuedQueries) { - this.queuedQueries = []; - } - - (this.queuedQueries as any).push(...queries); - } - - /** - * Commit all queued queries. - * NOTE: to be used in combination with enqueueSaveBlock and enqueueDeleteBlock. - * @return {void} - */ - public async commitQueuedQueries() { - if (!this.queuedQueries || this.queuedQueries.length === 0) { - return; - } - - this.logger.debug("Committing database transactions."); - - try { - await this.db.tx(t => t.batch(this.queuedQueries)); - } catch (error) { - this.logger.error(error); - - throw error; - } finally { - this.queuedQueries = null; - } - } - - /** - * Get a block. - * @param {Number} id - * @return {Block} - */ - public async getBlock(id) { - // TODO: caching the last 1000 blocks, in combination with `saveBlock` could help to optimise - const block = await this.db.blocks.findById(id); - - if (!block) { - return null; - } - - const transactions = await this.db.transactions.findByBlock(block.id); - - block.transactions = transactions.map(({ serialized }) => Transaction.deserialize(serialized.toString("hex"))); - - return new Block(block); - } - - /** - * Get the last block. - * @return {(Block|null)} - */ - public async getLastBlock() { - const block = await this.db.blocks.latest(); - - if (!block) { - return null; - } - - const transactions = await this.db.transactions.latestByBlock(block.id); - - block.transactions = transactions.map(({ serialized }) => Transaction.deserialize(serialized.toString("hex"))); - - return new Block(block); - } - - /** - * Get a transaction. - * @param {Number} id - * @return {Promise} - */ - public async getTransaction(id) { - return this.db.transactions.findById(id); - } - - /** - * Get common blocks for the given IDs. - * @param {Array} ids - * @return {Array} - */ - public async getCommonBlocks(ids) { - const state = app.resolve("state"); - let commonBlocks = state.getCommonBlocks(ids); - if (commonBlocks.length < ids.length) { - commonBlocks = await this.db.blocks.common(ids); - } - - return commonBlocks; - } - - /** - * Get forged transactions for the given IDs. - * @param {Array} ids - * @return {Array} - */ - public async getForgedTransactionsIds(ids) { - if (!ids.length) { - return []; - } - - const transactions = await this.db.transactions.forged(ids); - - return transactions.map(transaction => transaction.id); - } - - /** - * Get blocks for the given offset and limit. - * @param {Number} offset - * @param {Number} limit - * @return {Array} - */ - public async getBlocks(offset, limit) { - let blocks = []; - - // The functions below return matches in the range [start, end], including both ends. - const start = offset; - const end = offset + limit - 1; - - if (app.has("state")) { - blocks = app.resolve("state").getLastBlocksByHeight(start, end); - } - - if (blocks.length !== limit) { - blocks = await this.db.blocks.heightRange(start, end); - - await this.loadTransactionsForBlocks(blocks); - } - - return blocks; - } - - /** - * Get top count blocks ordered by height DESC. - * NOTE: Only used when trying to restore database integrity. The returned blocks may be unchained. - * @param {Number} count - * @return {Array} - */ - public async getTopBlocks(count) { - const blocks = await this.db.blocks.top(count); - - await this.loadTransactionsForBlocks(blocks); - - return blocks; - } - - /** - * Load all transactions for the given blocks - * @param {Array} blocks - * @return {void} - */ - public async loadTransactionsForBlocks(blocks) { - if (!blocks.length) { - return; - } - - const ids = blocks.map(block => block.id); - - let transactions = await this.db.transactions.latestByBlocks(ids); - transactions = transactions.map(tx => { - const data = Transaction.deserialize(tx.serialized.toString("hex")); - data.blockId = tx.blockId; - return data; - }); - - for (const block of blocks) { - if (block.numberOfTransactions > 0) { - block.transactions = transactions.filter(transaction => transaction.blockId === block.id); - } - } - } - - /** - * Get the 10 recent block ids. - * @return {[]String} - */ - public async getRecentBlockIds() { - const state = app.resolve("state"); - let blocks = state - .getLastBlockIds() - .reverse() - .slice(0, 10); - - if (blocks.length < 10) { - blocks = await this.db.blocks.recent(); - blocks = blocks.map(block => block.id); - } - - return blocks; - } - - /** - * Get the headers of blocks for the given offset and limit. - * @param {Number} offset - * @param {Number} limit - * @return {Array} - */ - public async getBlockHeaders(offset, limit) { - const blocks = await this.db.blocks.headers(offset, offset + limit); - - return blocks.map(block => Block.serialize(block)); - } - - /** - * Get the cache object - * @return {Cache} - */ - public getCache() { - return this.cache; - } - - /** - * Run all migrations. - * @return {void} - */ - public async __runMigrations() { - for (const migration of migrations) { - const { name } = path.parse(migration.file); - - if (name === "20180304100000-create-migrations-table") { - await this.query.none(migration); - } else { - const row = await this.db.migrations.findByName(name); - - if (row === null) { - this.logger.debug(`Migrating ${name}`); - - await this.query.none(migration); - - await this.db.migrations.create({ name }); - } - } - } - } - - /** - * Register all models. - * @return {void} - */ - public async __registerModels() { - for (const [key, Value] of Object.entries(require("./models"))) { - this.models[key.toLowerCase()] = new (Value as any)(this.pgp); - } - } - - /** - * Register the query builder. - * @return {void} - */ - public __registerQueryExecutor() { - this.query = new QueryExecutor(this); - } - - /** - * Register event listeners. - * @return {void} - */ - public __registerListeners() { - super.__registerListeners(); - - this.emitter.on("wallet.created.cold", async coldWallet => { - try { - const wallet = await this.db.wallets.findByAddress(coldWallet.address); - - if (wallet) { - Object.keys(wallet).forEach(key => { - if (["balance"].indexOf(key) !== -1) { - return; - } - - coldWallet[key] = key !== "voteBalance" ? wallet[key] : new Bignum(wallet[key]); - }); - } - } catch (err) { - this.logger.error(err); - } - }); - - this.emitter.once("shutdown", async () => { - if (!this.spvFinished) { - // Prevent dirty wallets to be saved when SPV didn't finish - this.walletManager.clear(); - } - }); - } -} diff --git a/packages/core-database-postgres/src/index.ts b/packages/core-database-postgres/src/index.ts index ff4bbde04b..b4d0f3e18c 100644 --- a/packages/core-database-postgres/src/index.ts +++ b/packages/core-database-postgres/src/index.ts @@ -1,4 +1,4 @@ -export * from "./connection"; +export * from "./postgres-connection"; export * from "./migrations"; export * from "./spv"; export * from "./models"; diff --git a/packages/core-database-postgres/src/plugin.ts b/packages/core-database-postgres/src/plugin.ts index 3ff476131a..2e1c39fd57 100644 --- a/packages/core-database-postgres/src/plugin.ts +++ b/packages/core-database-postgres/src/plugin.ts @@ -1,7 +1,8 @@ -import { DatabaseManager } from "@arkecosystem/core-database"; -import { Container, Logger } from "@arkecosystem/core-interfaces"; -import { PostgresConnection } from "./connection"; +import { + DatabaseManager, databaseServiceFactory, WalletManager} from "@arkecosystem/core-database"; +import { Container, Database, Logger } from "@arkecosystem/core-interfaces"; import { defaults } from "./defaults"; +import { PostgresConnection } from "./postgres-connection"; export const plugin: Container.PluginDescriptor = { pkg: require("../package.json"), @@ -11,13 +12,18 @@ export const plugin: Container.PluginDescriptor = { async register(container: Container.IContainer, options) { container.resolvePlugin("logger").info("Establishing Database Connection"); + const walletManager = new WalletManager(); + const databaseManager = container.resolvePlugin("databaseManager"); - return await databaseManager.makeConnection(new PostgresConnection(options)); + + const connection = await databaseManager.makeConnection(new PostgresConnection(options, walletManager)); + + return await databaseServiceFactory(options, walletManager, connection); }, async deregister(container: Container.IContainer, options) { container.resolvePlugin("logger").info("Closing Database Connection"); - const connection = container.resolvePlugin("database"); - return connection.disconnect(); + const databaseService = container.resolvePlugin("database"); + await databaseService.connection.disconnect(); }, }; diff --git a/packages/core-database-postgres/src/postgres-connection.ts b/packages/core-database-postgres/src/postgres-connection.ts new file mode 100644 index 0000000000..442a19eb98 --- /dev/null +++ b/packages/core-database-postgres/src/postgres-connection.ts @@ -0,0 +1,275 @@ +import { app } from "@arkecosystem/core-container"; +import { Database, EventEmitter, Logger } from "@arkecosystem/core-interfaces"; +import { roundCalculator } from "@arkecosystem/core-utils"; +import { models } from "@arkecosystem/crypto"; +import fs from "fs"; +import chunk from "lodash/chunk"; +import path from "path"; +import pgPromise from "pg-promise"; +import { migrations } from "./migrations"; +import { Model } from "./models"; +import { repositories } from "./repositories"; +import { MigrationsRepository } from "./repositories/migrations"; +import { SPV } from "./spv"; +import { QueryExecutor } from "./sql/query-executor"; +import { camelizeColumns } from "./utils"; + +export class PostgresConnection implements Database.IDatabaseConnection { + + public logger = app.resolvePlugin("logger"); + public models: { [key: string]: Model } = {}; + public query: QueryExecutor; + public db: any; + public blocksRepository: Database.IBlocksRepository; + public roundsRepository: Database.IRoundsRepository; + public transactionsRepository: Database.ITransactionsRepository; + public walletsRepository: Database.IWalletsRepository; + public pgp: any; + private emitter = app.resolvePlugin("event-emitter"); + private migrationsRepository : MigrationsRepository; + private cache: Map; + private queuedQueries: any[]; + + + public constructor(readonly options: any, private walletManager: Database.IWalletManager) { + + } + + + public async buildWallets(height: number) { + const spvPath = `${process.env.CORE_PATH_CACHE}/spv.json`; + + if (fs.existsSync(spvPath)) { + (fs as any).removeSync(spvPath); + + this.logger.info("Ark Core ended unexpectedly - resuming from where we left off :runner:"); + + return true; + } + + try { + const spv = new SPV(this.query, this.walletManager); + return await spv.build(height); + } catch (error) { + this.logger.error(error.stack); + } + + return false; + } + + public async commitQueuedQueries() { + if (!this.queuedQueries || this.queuedQueries.length === 0) { + return; + } + + this.logger.debug("Committing database transactions."); + + try { + await this.db.tx(t => t.batch(this.queuedQueries)); + } catch (error) { + this.logger.error(error); + + throw error; + } finally { + this.queuedQueries = null; + } + } + + public async connect() { + + this.emitter.emit(Database.DatabaseEvents.PRE_CONNECT); + const initialization = { + receive(data, result, e) { + camelizeColumns(pgp, data); + }, + extend(object) { + for (const repository of Object.keys(repositories)) { + object[repository] = new repositories[repository](object, pgp); + } + }, + }; + + const pgp = pgPromise({ ...this.options.initialization, ...initialization }); + + this.pgp = pgp; + this.db = this.pgp(this.options.connection); + } + + public async deleteBlock(block: models.Block) { + try { + const queries = [this.transactionsRepository.deleteByBlockId(block.data.id), this.blocksRepository.delete(block.data.id)]; + + await this.db.tx(t => t.batch(queries)); + } catch (error) { + this.logger.error(error.stack); + + throw error; + } + } + + public async disconnect() { + this.logger.debug("Disconnecting from database"); + this.emitter.emit(Database.DatabaseEvents.PRE_DISCONNECT); + + try { + await this.commitQueuedQueries(); + this.cache.clear(); + } catch (error) { + this.logger.warn("Issue in commiting blocks, database might be corrupted"); + this.logger.warn(error.message); + } + + await this.pgp.end(); + this.emitter.emit(Database.DatabaseEvents.POST_DISCONNECT); + this.logger.debug("Disconnected from database"); + } + + public enqueueDeleteBlock(block: models.Block): any { + const queries = [this.transactionsRepository.deleteByBlockId(block.data.id), this.blocksRepository.delete(block.data.id)]; + + this.enqueueQueries(queries); + } + + public enqueueDeleteRound(height: number): any { + const { round, nextRound, maxDelegates } = roundCalculator.calculateRound(height); + + if (nextRound === round + 1 && height >= maxDelegates) { + this.enqueueQueries([this.roundsRepository.delete(nextRound)]); + } + } + + public enqueueSaveBlock(block: models.Block): any { + const queries = [this.blocksRepository.insert(block.data)]; + + if (block.transactions.length > 0) { + queries.push(this.transactionsRepository.insert(block.transactions)); + } + + this.enqueueQueries(queries); + } + + public async make(): Promise { + if (this.db) { + throw new Error("Database connection already initialised"); + } + + this.logger.debug("Connecting to database"); + + this.queuedQueries = null; + this.cache = new Map(); + + try { + await this.connect(); + this.exposeRepositories(); + await this.registerQueryExecutor(); + await this.runMigrations(); + await this.registerModels(); + this.logger.debug("Connected to database."); + this.emitter.emit(Database.DatabaseEvents.POST_CONNECT); + + return this; + } catch (error) { + app.forceExit("Unable to connect to the database!", error); + } + + return null; + } + + public async saveBlock(block: models.Block) { + try { + const queries = [this.blocksRepository.insert(block.data)]; + + if (block.transactions.length > 0) { + queries.push(this.transactionsRepository.insert(block.transactions)); + } + + await this.db.tx(t => t.batch(queries)); + } catch (err) { + this.logger.error(err.message); + } + } + + public async saveWallets(wallets: any[], force?: boolean) { + if (force) { + // all wallets to be updated, performance is better without upsert + await this.walletsRepository.truncate(); + + try { + const chunks = chunk(wallets, 5000).map(c => this.walletsRepository.insert(c)); // this 5000 figure should be configurable... + await this.db.tx(t => t.batch(chunks)); + } catch (error) { + this.logger.error(error.stack); + } + } else { + // NOTE: The list of delegates is calculated in-memory against the WalletManager, + // so it is safe to perform the costly UPSERT non-blocking during round change only: + // 'await saveWallets(false)' -> 'saveWallets(false)' + try { + const queries = wallets.map(wallet => this.walletsRepository.updateOrCreate(wallet)); + await this.db.tx(t => t.batch(queries)); + } catch (error) { + this.logger.error(error.stack); + } + } + } + + /** + * Run all migrations. + * @return {void} + */ + + private async runMigrations() { + for (const migration of migrations) { + const { name } = path.parse(migration.file); + + if (name === "20180304100000-create-migrations-table") { + await this.query.none(migration); + } else { + const row = await this.migrationsRepository.findByName(name); + + if (row === null) { + this.logger.debug(`Migrating ${name}`); + + await this.query.none(migration); + + await this.migrationsRepository.insert({ name }); + } + } + } + } + + + /** + * Register all models. + * @return {void} + */ + private async registerModels() { + for (const [key, Value] of Object.entries(require("./models"))) { + this.models[key.toLowerCase()] = new (Value as any)(this.pgp); + } + } + + /** + * Register the query builder. + * @return {void} + */ + private registerQueryExecutor() { + this.query = new QueryExecutor(this); + } + + private enqueueQueries(queries) { + if (!this.queuedQueries) { + this.queuedQueries = []; + } + + (this.queuedQueries as any).push(...queries); + } + + private exposeRepositories() { + this.blocksRepository = this.db.blocks; + this.transactionsRepository = this.db.transactions; + this.roundsRepository = this.db.rounds; + this.walletsRepository = this.db.wallets; + this.migrationsRepository = this.db.migrations; + } +} diff --git a/packages/core-database-postgres/src/repositories/blocks.ts b/packages/core-database-postgres/src/repositories/blocks.ts index ce2438442f..a2e77a3c98 100644 --- a/packages/core-database-postgres/src/repositories/blocks.ts +++ b/packages/core-database-postgres/src/repositories/blocks.ts @@ -1,10 +1,11 @@ +import { Database } from "@arkecosystem/core-interfaces"; import { Block } from "../models"; import { queries } from "../queries"; import { Repository } from "./repository"; const { blocks: sql } = queries; -export class BlocksRepository extends Repository { +export class BlocksRepository extends Repository implements Database.IBlocksRepository { /** * Find a block by its ID. * @param {Number} id @@ -19,7 +20,8 @@ export class BlocksRepository extends Repository { * @return {Promise} */ public async count() { - return this.db.one(sql.count); + const { count } = await this.db.one(sql.count); + return count ; } /** diff --git a/packages/core-database-postgres/src/repositories/repository.ts b/packages/core-database-postgres/src/repositories/repository.ts index ed9b9c0558..63de711e58 100644 --- a/packages/core-database-postgres/src/repositories/repository.ts +++ b/packages/core-database-postgres/src/repositories/repository.ts @@ -1,6 +1,7 @@ +import { Database } from "@arkecosystem/core-interfaces"; import { Model } from "../models"; -export abstract class Repository { +export abstract class Repository implements Database.IRepository { protected model: Model; /** @@ -36,20 +37,20 @@ export abstract class Repository { /** * Create one or many instances of the related models. - * @param {Array|Object} item + * @param {Array|Object} items * @return {Promise} */ - public async create(item) { - return this.db.none(this.__insertQuery(item)); + public async insert(items) { + return this.db.none(this.__insertQuery(items)); } /** * Update one or many instances of the related models. - * @param {Array|Object} item + * @param {Array|Object} items * @return {Promise} */ - public async update(item) { - return this.db.none(this.__updateQuery(item)); + public async update(items) { + return this.db.none(this.__updateQuery(items)); } /** diff --git a/packages/core-database-postgres/src/repositories/rounds.ts b/packages/core-database-postgres/src/repositories/rounds.ts index 8d87146de5..06602185de 100644 --- a/packages/core-database-postgres/src/repositories/rounds.ts +++ b/packages/core-database-postgres/src/repositories/rounds.ts @@ -1,10 +1,11 @@ +import { Database } from "@arkecosystem/core-interfaces"; import { Round } from "../models"; import { queries } from "../queries"; import { Repository } from "./repository"; const { rounds: sql } = queries; -export class RoundsRepository extends Repository { +export class RoundsRepository extends Repository implements Database.IRoundsRepository { /** * Find a round by its ID. * @param {Number} round diff --git a/packages/core-database-postgres/src/repositories/transactions.ts b/packages/core-database-postgres/src/repositories/transactions.ts index 12e1d0f275..843f71b241 100644 --- a/packages/core-database-postgres/src/repositories/transactions.ts +++ b/packages/core-database-postgres/src/repositories/transactions.ts @@ -1,10 +1,11 @@ +import { Database } from "@arkecosystem/core-interfaces"; import { Transaction } from "../models"; import { queries } from "../queries"; import { Repository } from "./repository"; const { transactions: sql } = queries; -export class TransactionsRepository extends Repository { +export class TransactionsRepository extends Repository implements Database.ITransactionsRepository { /** * Find a transactions by its ID. * @param {String} id @@ -19,7 +20,7 @@ export class TransactionsRepository extends Repository { * @param {String} id * @return {Promise} */ - public async findByBlock(id) { + public async findByBlockId(id) { return this.db.manyOrNone(sql.findByBlock, { id }); } @@ -63,7 +64,7 @@ export class TransactionsRepository extends Repository { * @param {Number} id * @return {Promise} */ - public async deleteByBlock(id) { + public async deleteByBlockId(id) { return this.db.none(sql.deleteByBlock, { id }); } diff --git a/packages/core-database-postgres/src/repositories/wallets.ts b/packages/core-database-postgres/src/repositories/wallets.ts index 5028613389..a95d4eba19 100644 --- a/packages/core-database-postgres/src/repositories/wallets.ts +++ b/packages/core-database-postgres/src/repositories/wallets.ts @@ -1,10 +1,11 @@ +import { Database } from "@arkecosystem/core-interfaces"; import { Wallet } from "../models"; import { queries } from "../queries"; import { Repository } from "./repository"; const { wallets: sql } = queries; -export class WalletsRepository extends Repository { +export class WalletsRepository extends Repository implements Database.IWalletsRepository { /** * Get all of the wallets from the database. * @return {Promise} @@ -26,7 +27,7 @@ export class WalletsRepository extends Repository { * Get the count of wallets that have a negative balance. * @return {Promise} */ - public async findNegativeBalances() { + public async tallyWithNegativeBalance() { return this.db.oneOrNone(sql.findNegativeBalances); } @@ -34,7 +35,7 @@ export class WalletsRepository extends Repository { * Get the count of wallets that have a negative vote balance. * @return {Promise} */ - public async findNegativeVoteBalances() { + public async tallyWithNegativeVoteBalance() { return this.db.oneOrNone(sql.findNegativeVoteBalances); } diff --git a/packages/core-database-postgres/src/spv.ts b/packages/core-database-postgres/src/spv.ts index 8d165867dc..fdb1998618 100644 --- a/packages/core-database-postgres/src/spv.ts +++ b/packages/core-database-postgres/src/spv.ts @@ -2,8 +2,7 @@ import { Bignum, models } from "@arkecosystem/crypto"; const { Transaction } = models; import { app } from "@arkecosystem/core-container"; -import { Logger } from "@arkecosystem/core-interfaces"; -import { PostgresConnection } from "./connection"; +import { Database, Logger } from "@arkecosystem/core-interfaces"; import { queries } from "./queries"; import { QueryExecutor } from "./sql/query-executor"; @@ -13,16 +12,7 @@ const config = app.getConfig(); const genesisWallets = config.get("genesisBlock.transactions").map(tx => tx.senderId); export class SPV { - private models: any; - private walletManager: any; - private query: QueryExecutor; - private activeDelegates: []; - - constructor(connectionInterface: PostgresConnection) { - this.models = connectionInterface.models; - this.walletManager = connectionInterface.walletManager; - this.query = connectionInterface.query; - } + constructor(private query: QueryExecutor, private walletManager: Database.IWalletManager) {} /** * Perform the SPV (Simple Payment Verification). @@ -30,7 +20,6 @@ export class SPV { * @return {void} */ public async build(height) { - this.activeDelegates = config.getMilestone(height).activeDelegates; logger.info("SPV Step 1 of 8: Received Transactions"); await this.__buildReceivedTransactions(); @@ -56,8 +45,8 @@ export class SPV { logger.info("SPV Step 8 of 8: MultiSignatures"); await this.__buildMultisignatures(); - logger.info(`SPV rebuild finished, wallets in memory: ${Object.keys(this.walletManager.byAddress).length}`); - logger.info(`Number of registered delegates: ${Object.keys(this.walletManager.byUsername).length}`); + logger.info(`SPV rebuild finished, wallets in memory: ${Object.keys(this.walletManager.allByAddress()).length}`); + logger.info(`Number of registered delegates: ${Object.keys(this.walletManager.allByUsername()).length}`); return this.__verifyWalletsConsistency(); } @@ -200,7 +189,8 @@ export class SPV { delegates.forEach((delegate, i) => { const wallet = this.walletManager.findByPublicKey(delegate.publicKey); wallet.missedBlocks = +delegate.missedBlocks; - wallet.rate = i + 1; + // TODO: unknown property 'rate' being access on Wallet class + (wallet as any).rate = i + 1; this.walletManager.reindex(wallet); }); } diff --git a/packages/core-database-postgres/src/sql/query-executor.ts b/packages/core-database-postgres/src/sql/query-executor.ts index 64ec92b0b8..123be4115e 100644 --- a/packages/core-database-postgres/src/sql/query-executor.ts +++ b/packages/core-database-postgres/src/sql/query-executor.ts @@ -1,4 +1,4 @@ -import { PostgresConnection } from "../connection"; +import { PostgresConnection } from "../postgres-connection"; export class QueryExecutor { /** diff --git a/packages/core-database/__tests__/__fixtures__/database-connection-stub.ts b/packages/core-database/__tests__/__fixtures__/database-connection-stub.ts new file mode 100644 index 0000000000..34459b572c --- /dev/null +++ b/packages/core-database/__tests__/__fixtures__/database-connection-stub.ts @@ -0,0 +1,53 @@ +// tslint:disable:no-empty + +import { Database } from "@arkecosystem/core-interfaces"; +import { models } from "@arkecosystem/crypto"; + +export class DatabaseConnectionStub implements Database.IDatabaseConnection { + public blocksRepository: Database.IBlocksRepository; + public roundsRepository: Database.IRoundsRepository; + public transactionsRepository: Database.ITransactionsRepository; + public walletsRepository: Database.IWalletsRepository; + public options: any; + + public buildWallets(height: number): Promise { + return undefined; + } + + public commitQueuedQueries(): any { + } + + public connect(): Promise { + return undefined; + } + + public deleteBlock(block: models.Block): Promise { + return undefined; + } + + public disconnect(): Promise { + return undefined; + } + + public enqueueDeleteBlock(block: models.Block): any { + } + + public enqueueDeleteRound(height: number): any { + } + + public enqueueSaveBlock(block: models.Block): any { + return null; + } + + public async make(): Promise { + return this; + } + + public saveBlock(block: models.Block): Promise { + return undefined; + } + + public saveWallets(wallets: any[], force?: boolean): Promise { + return undefined; + } +} diff --git a/packages/core-database/__tests__/__fixtures__/dummy-class.ts b/packages/core-database/__tests__/__fixtures__/dummy-class.ts deleted file mode 100644 index 4350503a8c..0000000000 --- a/packages/core-database/__tests__/__fixtures__/dummy-class.ts +++ /dev/null @@ -1,71 +0,0 @@ -// tslint:disable:no-empty - -import { ConnectionInterface } from "../../src"; - -export class DummyConnection extends ConnectionInterface { - constructor(options: any) { - super(options); - } - - public async connect(): Promise {} - - public async disconnect(): Promise {} - - public async verifyBlockchain(): Promise { - return true; - } - - public async getActiveDelegates(height, delegates?): Promise { - return []; - } - - public async buildWallets(height): Promise { - return true; - } - - public async saveWallets(force): Promise {} - - public async saveBlock(block): Promise {} - - public enqueueSaveBlock(block): void {} - - public enqueueDeleteBlock(block): void {} - - public enqueueDeleteRound(height): void {} - - public async commitQueuedQueries(): Promise {} - - public async deleteBlock(block): Promise {} - - public async getBlock(id): Promise { - return true; - } - - public async getLastBlock(): Promise { - return true; - } - - public async getBlocks(offset, limit): Promise { - return []; - } - - public async getTopBlocks(count): Promise { - return []; - } - - public async getRecentBlockIds(): Promise { - return []; - } - - public async saveRound(activeDelegates): Promise {} - - public async deleteRound(round): Promise {} - - public async getTransaction(id): Promise { - return true; - } - - public async make(): Promise { - return this; - } -} diff --git a/packages/core-database/__tests__/__fixtures__/state-storage-stub.ts b/packages/core-database/__tests__/__fixtures__/state-storage-stub.ts new file mode 100644 index 0000000000..947a552faf --- /dev/null +++ b/packages/core-database/__tests__/__fixtures__/state-storage-stub.ts @@ -0,0 +1,56 @@ +/* tslint:disable:no-empty */ +import { Blockchain } from "@arkecosystem/core-interfaces"; +import { models } from "@arkecosystem/crypto"; + +export class StateStorageStub implements Blockchain.IStateStorage { + public cacheTransactions(transactions: models.ITransactionData[]): { added: models.ITransactionData[]; notAdded: models.ITransactionData[] } { + return undefined; + } + + public clear(): void { + } + + public clearWakeUpTimeout(): void { + } + + public getCachedTransactionIds(): string[] { + return []; + } + + public getCommonBlocks(ids: string[]): models.IBlockData[] { + return []; + } + + public getLastBlock(): models.Block | null { + return undefined; + } + + public getLastBlockIds(): string[] { + return []; + } + + public getLastBlocks(): models.Block[] { + return []; + } + + public getLastBlocksByHeight(start: number, end?: number): models.IBlockData[] { + return []; + } + + public pingBlock(incomingBlock: models.IBlockData): boolean { + return false; + } + + public pushPingBlock(block: models.IBlockData): void { + } + + public removeCachedTransactionIds(transactionIds: string[]): void { + } + + public reset(): void { + } + + public setLastBlock(block: models.Block): void { + } + +} diff --git a/packages/core-database/__tests__/__support__/setup.ts b/packages/core-database/__tests__/__support__/setup.ts index 4cfcc58d25..faba7ee97d 100644 --- a/packages/core-database/__tests__/__support__/setup.ts +++ b/packages/core-database/__tests__/__support__/setup.ts @@ -7,7 +7,7 @@ export const setUp = async () => { process.env.CORE_SKIP_BLOCKCHAIN = "true"; - await setUpContainer({ + return await setUpContainer({ exit: "@arkecosystem/core-blockchain", exclude: [ "@arkecosystem/core-p2p", diff --git a/packages/core-database/__tests__/database-service.test.ts b/packages/core-database/__tests__/database-service.test.ts new file mode 100644 index 0000000000..399397e000 --- /dev/null +++ b/packages/core-database/__tests__/database-service.test.ts @@ -0,0 +1,221 @@ +import { Container, Database, EventEmitter } from "@arkecosystem/core-interfaces"; +import { Bignum, constants, models, transactionBuilder } from "@arkecosystem/crypto"; +import "jest-extended"; +import { WalletManager } from "../src"; +import { DatabaseService } from "../src/database-service"; +import { DatabaseConnectionStub } from "./__fixtures__/database-connection-stub"; +import { StateStorageStub } from "./__fixtures__/state-storage-stub"; +import { setUp, tearDown } from "./__support__/setup"; + +const { Block, Transaction, Wallet } = models; + +const { ARKTOSHI, TransactionTypes } = constants; + +let connection : Database.IDatabaseConnection; +let databaseService : DatabaseService; +let walletManager : Database.IWalletManager; +let genesisBlock : models.Block; +let container: Container.IContainer; +let emitter : EventEmitter.EventEmitter; + + +beforeAll(async () => { + container = await setUp(); + emitter = container.resolvePlugin("event-emitter"); + genesisBlock = new Block(require("@arkecosystem/core-test-utils/src/config/testnet/genesisBlock.json")); + connection = new DatabaseConnectionStub(); + walletManager = new WalletManager(); +}); + +afterAll(async () => { + await tearDown(); +}); + +beforeEach(()=> { + jest.restoreAllMocks() +}); + +function createService() { + return new DatabaseService({}, connection, walletManager, null, null); +} + +describe('Database Service', () => { + it('should listen for emitter events during constructor', () => { + jest.spyOn(emitter, 'on'); + jest.spyOn(emitter, 'once'); + + databaseService = createService(); + + + expect(emitter.on).toHaveBeenCalledWith('state:started', expect.toBeFunction()); + expect(emitter.on).toHaveBeenCalledWith('wallet.created.cold', expect.toBeFunction()); + expect(emitter.once).toHaveBeenCalledWith('shutdown', expect.toBeFunction()); + }); + + describe('applyBlock', () => { + it('should applyBlock', async () => { + jest.spyOn(walletManager, 'applyBlock').mockImplementation( (block) => block ); + jest.spyOn(emitter, 'emit'); + + + databaseService = createService(); + jest.spyOn(databaseService, 'applyRound').mockImplementation(() => null); // test applyRound logic separately + + await databaseService.applyBlock(genesisBlock); + + + expect(walletManager.applyBlock).toHaveBeenCalledWith(genesisBlock); + expect(emitter.emit).toHaveBeenCalledWith('block.applied', genesisBlock.data); + genesisBlock.transactions.forEach(tx => expect(emitter.emit).toHaveBeenCalledWith('transaction.applied', tx.data)); + }) + }); + + describe('getBlocksForRound', () => { + it('should fetch blocks using lastBlock in state-storage', async() => { + const stateStorageStub = new StateStorageStub(); + jest.spyOn(stateStorageStub, 'getLastBlock').mockReturnValue(null); + jest.spyOn(container, 'has').mockReturnValue(true); + jest.spyOn(container, 'resolve').mockReturnValue(stateStorageStub); + + databaseService = createService(); + jest.spyOn(databaseService, 'getLastBlock').mockReturnValue(null); + + + const blocks = await databaseService.getBlocksForRound(); + + + expect(blocks).toBeEmpty(); + expect(stateStorageStub.getLastBlock).toHaveBeenCalled(); + expect(databaseService.getLastBlock).not.toHaveBeenCalled(); + + }); + + it('should fetch blocks using lastBlock in database', async () => { + jest.spyOn(container, 'has').mockReturnValue(false); + + databaseService = createService(); + jest.spyOn(databaseService, 'getLastBlock').mockReturnValue(null); + + + const blocks = await databaseService.getBlocksForRound(); + + + expect(blocks).toBeEmpty(); + expect(databaseService.getLastBlock).toHaveBeenCalled(); + }); + + it('should fetch blocks from lastBlock height', async () => { + databaseService = createService(); + + jest.spyOn(databaseService, 'getLastBlock').mockReturnValue(genesisBlock); + jest.spyOn(databaseService, 'getBlocks').mockReturnValue([]); + jest.spyOn(container, 'has').mockReturnValue(false); + + + const blocks = await databaseService.getBlocksForRound(); + + + expect(blocks).toBeEmpty(); + expect(databaseService.getBlocks).toHaveBeenCalledWith(1, container.getConfig().getMilestone(genesisBlock.data.height).activeDelegates); + }) + }); + + /* TODO: Testing a method that's private. This needs a replacement by testing a public method instead */ + + describe("calcPreviousActiveDelegates", () => { + it("should calculate the previous delegate list", async () => { + walletManager = new WalletManager(); + const initialHeight = 52; + + // Create delegates + for (const transaction of genesisBlock.transactions) { + if (transaction.type === TransactionTypes.DelegateRegistration) { + const wallet = walletManager.findByPublicKey(transaction.senderPublicKey); + wallet.username = Transaction.deserialize( + transaction.serialized.toString(), + ).asset.delegate.username; + walletManager.reindex(wallet); + } + } + + const keys = { + passphrase: "this is a secret passphrase", + publicKey: "02c71ab1a1b5b7c278145382eb0b535249483b3c4715a4fe6169d40388bbb09fa7", + privateKey: "dcf4ead2355090279aefba91540f32e93b15c541ecb48ca73071f161b4f3e2e3", + address: "D64cbDctaiADEH7NREnvRQGV27bnb1v2kE", + }; + + // Beginning of round 2 with all delegates 0 vote balance. + const delegatesRound2 = walletManager.loadActiveDelegateList(51, initialHeight); + + // Prepare sender wallet + const sender = new Wallet(keys.address); + sender.publicKey = keys.publicKey; + sender.canApply = jest.fn(() => true); + walletManager.reindex(sender); + + // Apply 51 blocks, where each increases the vote balance of a delegate to + // reverse the current delegate order. + const blocksInRound = []; + for (let i = 0; i < 51; i++) { + const transfer = transactionBuilder + .transfer() + .amount(i * ARKTOSHI) + .recipientId(delegatesRound2[i].address) + .sign(keys.passphrase) + .build(); + + // Vote for itself + walletManager.findByPublicKey(delegatesRound2[i].publicKey).vote = delegatesRound2[i].publicKey; + // walletManager.byPublicKey[delegatesRound2[i].publicKey].vote = delegatesRound2[i].publicKey; + + const block = Block.create( + { + version: 0, + timestamp: 0, + height: initialHeight + i, + numberOfTransactions: 1, + totalAmount: transfer.amount, + totalFee: new Bignum(0.1), + reward: new Bignum(2), + payloadLength: 0, + payloadHash: "a".repeat(64), + transactions: [transfer], + }, + keys, + ); + + block.data.generatorPublicKey = keys.publicKey; + walletManager.applyBlock(block); + + blocksInRound.push(block); + } + + // The delegates from round 2 are now reversed in rank in round 3. + const delegatesRound3 = walletManager.loadActiveDelegateList(51, initialHeight + 51); + for (let i = 0; i < delegatesRound3.length; i++) { + expect(delegatesRound3[i].rate).toBe(i + 1); + expect(delegatesRound3[i].publicKey).toBe(delegatesRound2[delegatesRound3.length - i - 1].publicKey); + } + + + jest.spyOn(databaseService, 'getBlocksForRound').mockReturnValue(blocksInRound); + databaseService.walletManager = walletManager; + + // Necessary for revertRound to not blow up. + walletManager.allByUsername = jest.fn(() => { + const usernames = Object.values((walletManager as any).byUsername); + usernames.push(sender); + return usernames; + }); + + // Finally recalculate the round 2 list and compare against the original list + const restoredDelegatesRound2 = await (databaseService as any).calcPreviousActiveDelegates(2); + + for (let i = 0; i < restoredDelegatesRound2.length; i++) { + expect(restoredDelegatesRound2[i].rate).toBe(i + 1); + expect(restoredDelegatesRound2[i].publicKey).toBe(delegatesRound2[i].publicKey); + } + }); + }); +}); diff --git a/packages/core-database/__tests__/interface.test.ts b/packages/core-database/__tests__/interface.test.ts deleted file mode 100644 index 7ee6c039c6..0000000000 --- a/packages/core-database/__tests__/interface.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -import "jest-extended"; - -import { Bignum, constants, models, transactionBuilder } from "@arkecosystem/crypto"; -import { setUp, tearDown } from "./__support__/setup"; - -const { Block, Transaction, Wallet } = models; - -const { ARKTOSHI, TransactionTypes } = constants; - -let connectionInterface; -let genesisBlock; - -import { DelegatesRepository } from "../src"; -import { WalletsRepository } from "../src"; -import { WalletManager } from "../src"; -import { DummyConnection } from "./__fixtures__/dummy-class"; - -beforeAll(async () => { - await setUp(); - - connectionInterface = new DummyConnection({}); - genesisBlock = new Block(require("@arkecosystem/core-test-utils/src/config/testnet/genesisBlock.json")); -}); - -afterAll(async () => { - await tearDown(); -}); - -describe("Connection Interface", () => { - describe("__calcPreviousActiveDelegates", () => { - it("should calculate the previous delegate list", async () => { - const walletManager = new WalletManager(); - const initialHeight = 52; - - // Create delegates - for (const transaction of genesisBlock.transactions) { - if (transaction.type === TransactionTypes.DelegateRegistration) { - const wallet = walletManager.findByPublicKey(transaction.senderPublicKey); - wallet.username = Transaction.deserialize( - transaction.serialized.toString("hex"), - ).asset.delegate.username; - walletManager.reindex(wallet); - } - } - - const keys = { - passphrase: "this is a secret passphrase", - publicKey: "02c71ab1a1b5b7c278145382eb0b535249483b3c4715a4fe6169d40388bbb09fa7", - privateKey: "dcf4ead2355090279aefba91540f32e93b15c541ecb48ca73071f161b4f3e2e3", - address: "D64cbDctaiADEH7NREnvRQGV27bnb1v2kE", - }; - - // Beginning of round 2 with all delegates 0 vote balance. - const delegatesRound2 = walletManager.loadActiveDelegateList(51, initialHeight); - - // Prepare sender wallet - const sender = new Wallet(keys.address); - sender.publicKey = keys.publicKey; - sender.canApply = jest.fn(() => true); - walletManager.reindex(sender); - - // Apply 51 blocks, where each increases the vote balance of a delegate to - // reverse the current delegate order. - const blocksInRound = []; - for (let i = 0; i < 51; i++) { - const transfer = transactionBuilder - .transfer() - .amount(i * ARKTOSHI) - .recipientId(delegatesRound2[i].address) - .sign(keys.passphrase) - .build(); - - // Vote for itself - walletManager.byPublicKey[delegatesRound2[i].publicKey].vote = delegatesRound2[i].publicKey; - - const block = Block.create( - { - version: 0, - timestamp: 0, - height: initialHeight + i, - numberOfTransactions: 1, - totalAmount: transfer.amount, - totalFee: new Bignum(0.1), - reward: new Bignum(2), - payloadLength: 0, - payloadHash: "a".repeat(64), - transactions: [transfer], - }, - keys, - ); - - block.data.generatorPublicKey = keys.publicKey; - walletManager.applyBlock(block); - - blocksInRound.push(block); - } - - // The delegates from round 2 are now reversed in rank in round 3. - const delegatesRound3 = walletManager.loadActiveDelegateList(51, initialHeight + 51); - for (let i = 0; i < delegatesRound3.length; i++) { - expect(delegatesRound3[i].rate).toBe(i + 1); - expect(delegatesRound3[i].publicKey).toBe(delegatesRound2[delegatesRound3.length - i - 1].publicKey); - } - - const connection = new DummyConnection({}); - connection.__getBlocksForRound = jest.fn(async () => blocksInRound); - connection.walletManager = walletManager; - - // Necessary for revertRound to not blow up. - walletManager.allByUsername = jest.fn(() => { - const usernames = Object.values(walletManager.byUsername); - usernames.push(sender); - return usernames; - }); - - // Finally recalculate the round 2 list and compare against the original list - const restoredDelegatesRound2 = await connection.__calcPreviousActiveDelegates(2); - - for (let i = 0; i < restoredDelegatesRound2.length; i++) { - expect(restoredDelegatesRound2[i].rate).toBe(i + 1); - expect(restoredDelegatesRound2[i].publicKey).toBe(delegatesRound2[i].publicKey); - } - }); - }); - - describe("_registerWalletManager", () => { - it("should register the wallet manager", () => { - expect(connectionInterface.walletManager).toBeNull(); - - connectionInterface._registerWalletManager(); - - expect(connectionInterface.walletManager).toBeInstanceOf(WalletManager); - }); - }); - - describe("_registerRepositories", () => { - it("should register the repositories", async () => { - expect(connectionInterface.wallets).toBeNull(); - expect(connectionInterface.delegates).toBeNull(); - - connectionInterface._registerRepositories(); - - expect(connectionInterface.wallets).toBeInstanceOf(WalletsRepository); - expect(connectionInterface.delegates).toBeInstanceOf(DelegatesRepository); - }); - }); -}); diff --git a/packages/core-database/__tests__/repositories/delegates.test.ts b/packages/core-database/__tests__/repositories/delegates.test.ts index 2df8740b31..1470ae85ce 100644 --- a/packages/core-database/__tests__/repositories/delegates.test.ts +++ b/packages/core-database/__tests__/repositories/delegates.test.ts @@ -1,8 +1,9 @@ +import { Database } from "@arkecosystem/core-interfaces"; +import { delegateCalculator } from "@arkecosystem/core-utils"; import { Bignum, constants, crypto, models } from "@arkecosystem/crypto"; import genesisBlockTestnet from "../../../core-test-utils/src/config/testnet/genesisBlock.json"; - -import { delegateCalculator } from "@arkecosystem/core-utils"; -import { DelegatesRepository } from "../../src/repositories/delegates"; +import { DelegatesRepository, WalletsRepository } from "../../src"; +import { DatabaseService } from "../../src/database-service"; import { setUp, tearDown } from "../__support__/setup"; const { ARKTOSHI } = constants; @@ -10,7 +11,10 @@ const { Block } = models; let genesisBlock; let repository; -let walletManager; + +let walletsRepository : Database.IWalletsBusinessRepository; +let walletManager: Database.IWalletManager; +let databaseService: Database.IDatabaseService; beforeAll(async done => { await setUp(); @@ -32,9 +36,9 @@ beforeEach(async done => { const { WalletManager } = require("../../src/wallet-manager"); walletManager = new WalletManager(); - repository = new DelegatesRepository({ - walletManager, - }); + repository = new DelegatesRepository(() => databaseService); + walletsRepository = new WalletsRepository(() => databaseService); + databaseService = new DatabaseService(null, null, walletManager, walletsRepository, repository); done(); }); @@ -62,10 +66,12 @@ describe("Delegate Repository", () => { const wallets = [delegates[0], {}, delegates[1], { username: "" }, delegates[2], {}]; it("should return the local wallets of the connection that are delegates", () => { - repository.connection.walletManager.all = jest.fn(() => wallets); + jest.spyOn(walletManager, 'allByAddress').mockReturnValue(wallets); + + const actualDelegates = repository.getLocalDelegates(); - expect(repository.getLocalDelegates()).toEqual(expect.arrayContaining(delegates)); - expect(repository.connection.walletManager.all).toHaveBeenCalled(); + expect(actualDelegates).toEqual(expect.arrayContaining(delegates)); + expect(walletManager.allByAddress).toHaveBeenCalled(); }); }); @@ -118,55 +124,6 @@ describe("Delegate Repository", () => { }); }); - describe("paginate", () => { - it("should be ok without params", () => { - const wallets = generateWallets(); - walletManager.index(wallets); - - const { count, rows } = repository.paginate(); - expect(count).toBe(52); - expect(rows).toHaveLength(52); - expect(rows.sort((a, b) => a.rate < b.rate)).toEqual(rows); - }); - - it("should be ok with params", () => { - const wallets = generateWallets(); - walletManager.index(wallets); - - const { count, rows } = repository.paginate({ offset: 10, limit: 10, orderBy: "rate:desc" }); - expect(count).toBe(52); - expect(rows).toHaveLength(10); - expect(rows.sort((a, b) => a.rate > b.rate)).toEqual(rows); - }); - - it("should be ok with params (no offset)", () => { - const wallets = generateWallets(); - walletManager.index(wallets); - - const { count, rows } = repository.paginate({ limit: 10 }); - expect(count).toBe(52); - expect(rows).toHaveLength(10); - }); - - it("should be ok with params (offset = 0)", () => { - const wallets = generateWallets(); - walletManager.index(wallets); - - const { count, rows } = repository.paginate({ offset: 0, limit: 12 }); - expect(count).toBe(52); - expect(rows).toHaveLength(12); - }); - - it("should be ok with params (no limit)", () => { - const wallets = generateWallets(); - walletManager.index(wallets); - - const { count, rows } = repository.paginate({ offset: 10 }); - expect(count).toBe(52); - expect(rows).toHaveLength(42); - }); - }); - describe("search", () => { beforeEach(() => { const wallets = generateWallets(); @@ -193,8 +150,7 @@ describe("Delegate Repository", () => { describe('when a username is "undefined"', () => { it("should return it", () => { // Index a wallet with username "undefined" - const address = Object.keys(walletManager.byAddress)[0]; - walletManager.byAddress[address].username = "undefined"; + walletManager.allByAddress()[0].username = 'undefined'; const username = "undefined"; const { count, rows } = repository.search({ username }); @@ -266,8 +222,7 @@ describe("Delegate Repository", () => { describe('when a username is "undefined"', () => { it("should return all results", () => { // Index a wallet with username "undefined" - const address = Object.keys(walletManager.byAddress)[0]; - walletManager.byAddress[address].username = "undefined"; + walletManager.allByAddress()[0].username = "undefined"; const { count, rows } = repository.search({}); expect(count).toBe(52); @@ -303,7 +258,7 @@ describe("Delegate Repository", () => { }); describe("getActiveAtHeight", () => { - it("should be ok", () => { + it("should be ok", async () => { const wallets = generateWallets(); walletManager.index(wallets); @@ -316,12 +271,10 @@ describe("Delegate Repository", () => { }; const height = 1; - repository.connection.getActiveDelegates = jest.fn(() => [delegate]); - repository.connection.wallets = { - findById: jest.fn(() => delegate), - }; + jest.spyOn(databaseService, 'getActiveDelegates').mockReturnValue([delegate]); + jest.spyOn(walletsRepository, 'findById').mockReturnValue(delegate); - const results = repository.getActiveAtHeight(height); + const results = await repository.getActiveAtHeight(height); expect(results).toBeArray(); expect(results[0].username).toBeString(); diff --git a/packages/core-database/__tests__/repositories/wallets.test.ts b/packages/core-database/__tests__/repositories/wallets.test.ts index 157c350718..2ddea64513 100644 --- a/packages/core-database/__tests__/repositories/wallets.test.ts +++ b/packages/core-database/__tests__/repositories/wallets.test.ts @@ -1,17 +1,20 @@ +import { Database } from "@arkecosystem/core-interfaces"; import { Bignum, crypto, models } from "@arkecosystem/crypto"; import compact from "lodash/compact"; import uniq from "lodash/uniq"; import genesisBlockTestnet from "../../../core-test-utils/src/config/testnet/genesisBlock.json"; import { setUp, tearDown } from "../__support__/setup"; -import { WalletsRepository } from "../../src/repositories/wallets"; +import { WalletsRepository } from "../../src"; +import { DatabaseService } from "../../src/database-service"; -const { Block } = models; +const { Block, Wallet } = models; let genesisBlock; let genesisSenders; let repository; -let walletManager; +let walletManager: Database.IWalletManager; +let databaseService: Database.IDatabaseService; beforeAll(async done => { await setUp(); @@ -34,9 +37,9 @@ beforeEach(async done => { const { WalletManager } = require("../../src/wallet-manager"); walletManager = new WalletManager(); - repository = new WalletsRepository({ - walletManager, - }); + repository = new WalletsRepository(() => databaseService); + + databaseService = new DatabaseService(null, null, walletManager, repository, null); done(); }); @@ -73,9 +76,11 @@ function generateFullWallets() { describe("Wallet Repository", () => { describe("all", () => { it("should return the local wallets of the connection", () => { - repository.connection.walletManager.all = jest.fn(); + jest.spyOn(walletManager, 'allByAddress').mockReturnValue(null); + repository.all(); - expect(repository.connection.walletManager.all).toHaveBeenCalled(); + + expect(walletManager.allByAddress).toHaveBeenCalled(); }); }); @@ -211,10 +216,17 @@ describe("Wallet Repository", () => { }); describe("top", () => { + beforeEach(() => { - walletManager.reindex({ address: "dummy-1", balance: new Bignum(1000) }); - walletManager.reindex({ address: "dummy-2", balance: new Bignum(2000) }); - walletManager.reindex({ address: "dummy-3", balance: new Bignum(3000) }); + [ + { address: 'dummy-1', balance: new Bignum(1000) }, + { address: 'dummy-2', balance: new Bignum(2000) }, + { address: 'dummy-3', balance: new Bignum(3000) }, + ].forEach(o => { + const wallet = new Wallet(o.address); + wallet.balance = o.balance; + walletManager.reindex(wallet); + }); }); it("should be ok without params", () => { diff --git a/packages/core-database/__tests__/wallet-manager.test.ts b/packages/core-database/__tests__/wallet-manager.test.ts index 53b7454907..b28f908cc9 100644 --- a/packages/core-database/__tests__/wallet-manager.test.ts +++ b/packages/core-database/__tests__/wallet-manager.test.ts @@ -1,4 +1,5 @@ /* tslint:disable:max-line-length no-empty */ +import { Database } from "@arkecosystem/core-interfaces"; import { fixtures, generators } from "@arkecosystem/core-test-utils"; import { Bignum, constants, crypto, models, transactionBuilder } from "@arkecosystem/crypto"; import { IMultiSignatureAsset } from "@arkecosystem/crypto/dist/models"; @@ -18,7 +19,7 @@ const walletData1 = wallets[0]; const walletData2 = wallets[1]; let genesisBlock; -let walletManager; +let walletManager : Database.IWalletManager; beforeAll(async done => { await setUp(); @@ -50,10 +51,10 @@ describe("Wallet Manager", () => { const wallet = new Wallet(walletData1.address); walletManager.reindex(wallet); - expect(walletManager.all()).toEqual([wallet]); + expect(walletManager.allByAddress()).toEqual([wallet]); walletManager.reset(); - expect(walletManager.all()).toEqual([]); + expect(walletManager.allByAddress()).toEqual([]); }); }); @@ -61,10 +62,10 @@ describe("Wallet Manager", () => { it("should index the wallets", () => { const wallet = new Wallet(walletData1.address); - expect(walletManager.all()).toEqual([]); + expect(walletManager.allByAddress()).toEqual([]); walletManager.reindex(wallet); - expect(walletManager.all()).toEqual([wallet]); + expect(walletManager.allByAddress()).toEqual([wallet]); }); }); @@ -85,9 +86,9 @@ describe("Wallet Manager", () => { beforeEach(() => { delegateMock = { applyBlock: jest.fn(), publicKey: delegatePublicKey }; - walletManager.findByPublicKey = jest.fn(() => delegateMock); - walletManager.applyTransaction = jest.fn(); - walletManager.revertTransaction = jest.fn(); + jest.spyOn(walletManager, 'findByPublicKey').mockReturnValue(delegateMock); + jest.spyOn(walletManager, 'applyTransaction').mockImplementation(); + jest.spyOn(walletManager, 'revertTransaction').mockImplementation(); const { data } = block; data.transactions = []; @@ -103,7 +104,7 @@ describe("Wallet Manager", () => { await walletManager.applyBlock(block2); block2.transactions.forEach((transaction, i) => { - expect(walletManager.applyTransaction.mock.calls[i][0]).toBe(block2.transactions[i]); + expect(walletManager.applyTransaction).toHaveBeenNthCalledWith(i+1, block2.transactions[i]) }); }); @@ -115,8 +116,8 @@ describe("Wallet Manager", () => { describe("when 1 transaction fails while applying it", () => { it("should revert sequentially (from last to first) all the transactions of the block", async () => { - walletManager.applyTransaction = jest.fn(transaction => { - if (transaction === block2.transactions[2]) { + jest.spyOn(walletManager, 'applyTransaction').mockImplementation( (tx) => { + if (tx === block2.transactions[2]) { throw new Error("Fake error"); } }); @@ -129,14 +130,14 @@ describe("Wallet Manager", () => { expect(null).toBe("this should fail if no error is thrown"); } catch (error) { expect(walletManager.revertTransaction).toHaveBeenCalledTimes(2); - block2.transactions.slice(0, 1).forEach((transaction, i) => { - expect(walletManager.revertTransaction.mock.calls[1 - i][0]).toEqual(block2.transactions[i]); + block2.transactions.slice(0, 1).forEach((transaction, i, total) => { + expect(walletManager.revertTransaction).toHaveBeenNthCalledWith(total.length+1 - i, block2.transactions[i]); }); } }); it("throws the Error", async () => { - walletManager.applyTransaction = jest.fn(transaction => { + walletManager.applyTransaction = jest.fn(tx => { throw new Error("Fake error"); }); @@ -177,7 +178,7 @@ describe("Wallet Manager", () => { const vote = generateVote("testnet", Math.random().toString(36), walletData2.publicKey, 1)[0]; describe.each` type | transaction | amount | balanceSuccess | balanceFail - ${"transfer"} | ${transfer} | ${new Bignum(96579)} | ${new Bignum(1 * ARKTOSHI)} | ${Bignum.ONE} + ${"transfer"} | ${transfer} | ${new Bignum(96579)} | ${new Bignum(ARKTOSHI)} | ${Bignum.ONE} ${"delegate"} | ${delegateReg} | ${Bignum.ZERO} | ${new Bignum(30 * ARKTOSHI)} | ${Bignum.ONE} ${"2nd sign"} | ${secondSign} | ${Bignum.ZERO} | ${new Bignum(10 * ARKTOSHI)} | ${Bignum.ONE} ${"vote"} | ${vote} | ${Bignum.ZERO} | ${new Bignum(5 * ARKTOSHI)} | ${Bignum.ONE} @@ -195,7 +196,7 @@ describe("Wallet Manager", () => { walletManager.reindex(sender); walletManager.reindex(recipient); - walletManager.__isDelegate = jest.fn(() => true); // for vote transaction + jest.spyOn(walletManager, 'isDelegate').mockReturnValue(true); }); it("should apply the transaction to the sender & recipient", async () => { @@ -237,7 +238,7 @@ describe("Wallet Manager", () => { it("should revert the transaction from the sender & recipient", async () => { const transaction = new Transaction({ type: TransactionTypes.Transfer, - amount: 245098000000000, + amount: new Bignum(245098000000000), fee: 0, recipientId: "AHXtmB84sTZ9Zd35h9Y1vfFvPE2Xzqj8ri", timestamp: 0, @@ -250,7 +251,7 @@ describe("Wallet Manager", () => { const sender = walletManager.findByPublicKey(transaction.data.senderPublicKey); const recipient = walletManager.findByAddress(transaction.data.recipientId); - recipient.balance = transaction.data.amount; + recipient.balance = new Bignum(transaction.data.amount); expect(sender.balance).toEqual(Bignum.ZERO); expect(recipient.balance).toEqual(transaction.data.amount); @@ -263,13 +264,6 @@ describe("Wallet Manager", () => { }); describe("findByAddress", () => { - it("should index it by address", () => { - const wallet = new Wallet(walletData1.address); - - walletManager.reindex(wallet); - expect(walletManager.byAddress[wallet.address]).toBe(wallet); - }); - it("should return it by address", () => { const wallet = new Wallet(walletData1.address); @@ -279,14 +273,6 @@ describe("Wallet Manager", () => { }); describe("findByPublicKey", () => { - it("should index it by publicKey", () => { - const wallet = new Wallet(walletData1.address); - wallet.publicKey = walletData1.publicKey; - - walletManager.reindex(wallet); - expect(walletManager.byPublicKey[wallet.publicKey]).toBe(wallet); - }); - it("should return it by publicKey", () => { const wallet = new Wallet(walletData1.address); wallet.publicKey = "dummy-public-key"; @@ -297,14 +283,6 @@ describe("Wallet Manager", () => { }); describe("findByUsername", () => { - it("should index it by username", () => { - const wallet = new Wallet(walletData1.address); - wallet.username = "dummy-username"; - - walletManager.reindex(wallet); - expect(walletManager.byUsername[wallet.username]).toBe(wallet); - }); - it("should return it by username", () => { const wallet = new Wallet(walletData1.address); wallet.username = "dummy-username"; @@ -322,15 +300,15 @@ describe("Wallet Manager", () => { const wallet2 = new Wallet(walletData2.address); walletManager.reindex(wallet2); - expect(walletManager.all()).toEqual([wallet1, wallet2]); + expect(walletManager.allByAddress()).toEqual([wallet1, wallet2]); }); }); - describe("__canBePurged", () => { + describe("canBePurged", () => { it("should be removed if all criteria are satisfied", async () => { const wallet = new Wallet(walletData1.address); - expect(walletManager.__canBePurged(wallet)).toBeTrue(); + expect(walletManager.canBePurged(wallet)).toBeTrue(); }); it("should not be removed if wallet.secondPublicKey is set", async () => { @@ -338,7 +316,7 @@ describe("Wallet Manager", () => { wallet.secondPublicKey = "secondPublicKey"; expect(wallet.secondPublicKey).toBe("secondPublicKey"); - expect(walletManager.__canBePurged(wallet)).toBeFalse(); + expect(walletManager.canBePurged(wallet)).toBeFalse(); }); it("should not be removed if wallet.multisignature is set", async () => { @@ -346,7 +324,7 @@ describe("Wallet Manager", () => { wallet.multisignature = {} as IMultiSignatureAsset; expect(wallet.multisignature).toEqual({}); - expect(walletManager.__canBePurged(wallet)).toBeFalse(); + expect(walletManager.canBePurged(wallet)).toBeFalse(); }); it("should not be removed if wallet.username is set", async () => { @@ -354,7 +332,7 @@ describe("Wallet Manager", () => { wallet.username = "username"; expect(wallet.username).toBe("username"); - expect(walletManager.__canBePurged(wallet)).toBeFalse(); + expect(walletManager.canBePurged(wallet)).toBeFalse(); }); }); @@ -371,7 +349,7 @@ describe("Wallet Manager", () => { walletManager.purgeEmptyNonDelegates(); - expect(walletManager.all()).toEqual([wallet2]); + expect(walletManager.allByAddress()).toEqual([wallet2]); }); it("should not be purged if wallet.secondPublicKey is set", async () => { @@ -387,7 +365,7 @@ describe("Wallet Manager", () => { walletManager.purgeEmptyNonDelegates(); - expect(walletManager.all()).toEqual([wallet1, wallet2]); + expect(walletManager.allByAddress()).toEqual([wallet1, wallet2]); }); it("should not be purged if wallet.multisignature is set", async () => { @@ -403,7 +381,7 @@ describe("Wallet Manager", () => { walletManager.purgeEmptyNonDelegates(); - expect(walletManager.all()).toEqual([wallet1, wallet2]); + expect(walletManager.allByAddress()).toEqual([wallet1, wallet2]); }); it("should not be purged if wallet.username is set", async () => { @@ -419,7 +397,7 @@ describe("Wallet Manager", () => { walletManager.purgeEmptyNonDelegates(); - expect(walletManager.all()).toEqual([wallet1, wallet2]); + expect(walletManager.allByAddress()).toEqual([wallet1, wallet2]); }); }); @@ -427,19 +405,15 @@ describe("Wallet Manager", () => { it("should update vote balance of delegates", async () => { for (let i = 0; i < 5; i++) { const delegateKey = i.toString().repeat(66); - const delegate = { - address: crypto.getAddress(delegateKey), - publicKey: delegateKey, - username: `delegate${i}`, - voteBalance: Bignum.ZERO, - }; - - const voter = { - address: crypto.getAddress((i + 5).toString().repeat(66)), - balance: new Bignum((i + 1) * 1000 * ARKTOSHI), - publicKey: `v${delegateKey}`, - vote: delegateKey, - }; + const delegate = new Wallet(crypto.getAddress(delegateKey)); + delegate.publicKey = delegateKey; + delegate.username = `delegate${i}`; + delegate.voteBalance = Bignum.ZERO; + + const voter = new Wallet(crypto.getAddress((i + 5).toString().repeat(66))); + voter.balance = new Bignum((i + 1) * 1000 * ARKTOSHI); + voter.publicKey = `v${delegateKey}`; + voter.vote = delegateKey; walletManager.index([delegate, voter]); } diff --git a/packages/core-database/src/database-service-factory.ts b/packages/core-database/src/database-service-factory.ts new file mode 100644 index 0000000000..daeceb6d20 --- /dev/null +++ b/packages/core-database/src/database-service-factory.ts @@ -0,0 +1,13 @@ +import { Database } from "@arkecosystem/core-interfaces"; +import { DatabaseService } from "./database-service"; +import { DelegatesRepository } from "./repositories/delegates"; +import { WalletsRepository } from "./repositories/wallets"; + +// Allow extenders of core-database to provide, optionally, a IWalletManager concrete in addition to a IDatabaseConnection, but keep the business repos common +export const databaseServiceFactory = async (opts: any, walletManager: Database.IWalletManager, connection: Database.IDatabaseConnection): Promise => { + let databaseService: DatabaseService; + databaseService = new DatabaseService(opts, connection, walletManager, new WalletsRepository(() => databaseService), new DelegatesRepository(() => databaseService)); + await databaseService.init(); + return databaseService; +}; + diff --git a/packages/core-database/src/database-service.ts b/packages/core-database/src/database-service.ts new file mode 100644 index 0000000000..cf95461a88 --- /dev/null +++ b/packages/core-database/src/database-service.ts @@ -0,0 +1,556 @@ +import { app } from "@arkecosystem/core-container"; +import { Blockchain, Database, EventEmitter, Logger } from "@arkecosystem/core-interfaces"; +import { roundCalculator } from "@arkecosystem/core-utils"; +import { Bignum, constants, crypto as arkCrypto, models } from "@arkecosystem/crypto"; +import assert from "assert"; +import crypto from "crypto"; +import cloneDeep from "lodash/cloneDeep"; +import pluralize from "pluralize"; +import { WalletManager } from "./wallet-manager"; + +const { Block, Transaction } = models; +const { TransactionTypes } = constants; + + +export class DatabaseService implements Database.IDatabaseService { + + public connection: Database.IDatabaseConnection; + public walletManager: Database.IWalletManager; + public logger = app.resolvePlugin("logger"); + public emitter = app.resolvePlugin("event-emitter"); + public config = app.getConfig(); + public options: any; + public wallets: Database.IWalletsBusinessRepository; + public delegates: Database.IDelegatesBusinessRepository; + public blocksInCurrentRound: any[] = null; + public stateStarted: boolean = false; + public restoredDatabaseIntegrity: boolean = false; + public forgingDelegates: any[] = null; + public cache: Map = new Map(); + private spvFinished: boolean; + + constructor(options: any, + connection: Database.IDatabaseConnection, + walletManager: Database.IWalletManager, + walletsBusinessRepository: Database.IWalletsBusinessRepository, + delegatesBusinessRepository: Database.IDelegatesBusinessRepository + ) { + this.connection = connection; + this.walletManager = walletManager; + this.options = options; + this.wallets = walletsBusinessRepository; + this.delegates = delegatesBusinessRepository; + + this.registerListeners(); + } + + public async init() { + await this.loadBlocksFromCurrentRound(); + } + + public async applyBlock(block: models.Block) { + this.walletManager.applyBlock(block); + + if (this.blocksInCurrentRound) { + this.blocksInCurrentRound.push(block); + } + + await this.applyRound(block.data.height); + block.transactions.forEach(tx => this.emitTransactionEvents(tx)); + this.emitter.emit("block.applied", block.data); + return true; + } + + public async applyRound(height: number) { + const nextHeight = height === 1 ? 1 : height + 1; + const maxDelegates = this.config.getMilestone(nextHeight).activeDelegates; + + if (nextHeight % maxDelegates === 1) { + const round = Math.floor((nextHeight - 1) / maxDelegates) + 1; + + if ( + !this.forgingDelegates || + this.forgingDelegates.length === 0 || + (this.forgingDelegates.length && this.forgingDelegates[0].round !== round) + ) { + this.logger.info(`Starting Round ${round.toLocaleString()} :dove_of_peace:`); + + try { + this.updateDelegateStats(this.forgingDelegates); + await this.saveWallets(false); // save only modified wallets during the last round + const delegates = this.walletManager.loadActiveDelegateList(maxDelegates, nextHeight); // get active delegate list from in-memory wallet manager + await this.saveRound(delegates); // save next round delegate list non-blocking + this.forgingDelegates = await this.getActiveDelegates(nextHeight, delegates); // generate the new active delegates list + this.blocksInCurrentRound.length = 0; + } catch (error) { + // trying to leave database state has it was + await this.deleteRound(round); + throw error; + } + } else { + this.logger.warn( + // tslint:disable-next-line:max-line-length + `Round ${round.toLocaleString()} has already been applied. This should happen only if you are a forger. :warning:`, + ); + } + } + } + + public async buildWallets(height: number): Promise { + this.walletManager.reset(); + + try { + const success = await this.connection.buildWallets(height); + this.spvFinished = true; + return success; + } catch (e) { + this.logger.error(e.stack); + } + return false; + } + + public async commitQueuedQueries() { + await this.connection.commitQueuedQueries(); + } + + public async deleteBlock(block: models.Block) { + await this.connection.deleteBlock(block); + } + + public async deleteRound(round: number) { + await this.connection.roundsRepository.delete(round); + } + + public enqueueDeleteBlock(block: models.Block) { + this.connection.enqueueDeleteBlock(block); + } + + public enqueueDeleteRound(height: number) { + this.connection.enqueueDeleteRound(height); + } + + public enqueueSaveBlock(block: models.Block) { + this.connection.enqueueSaveBlock(block); + } + + public async getActiveDelegates(height: number, delegates?: any[]) { + const maxDelegates = this.config.getMilestone(height).activeDelegates; + const round = Math.floor((height - 1) / maxDelegates) + 1; + + if (this.forgingDelegates && this.forgingDelegates.length && this.forgingDelegates[0].round === round) { + return this.forgingDelegates; + } + + // When called during applyRound we already know the delegates, so we don't have to query the database. + if (!delegates || delegates.length === 0) { + delegates = await this.connection.roundsRepository.findById(round); + } + + const seedSource = round.toString(); + let currentSeed = crypto + .createHash("sha256") + .update(seedSource, "utf8") + .digest(); + + for (let i = 0, delCount = delegates.length; i < delCount; i++) { + for (let x = 0; x < 4 && i < delCount; i++, x++) { + const newIndex = currentSeed[x] % delCount; + const b = delegates[newIndex]; + delegates[newIndex] = delegates[i]; + delegates[i] = b; + } + currentSeed = crypto + .createHash("sha256") + .update(currentSeed) + .digest(); + } + + this.forgingDelegates = delegates.map(delegate => { + delegate.round = +delegate.round; + return delegate; + }); + + return this.forgingDelegates; + } + + public async getBlock(id: string) { + // TODO: caching the last 1000 blocks, in combination with `saveBlock` could help to optimise + const block = await this.connection.blocksRepository.findById(id); + + if (!block) { + return null; + } + + const transactions = await this.connection.transactionsRepository.findByBlockId(block.id); + + block.transactions = transactions.map(({ serialized }) => Transaction.deserialize(serialized.toString("hex"))); + + return new Block(block); + } + + public async getBlocks(offset: number, limit: number) { + let blocks = []; + + // The functions below return matches in the range [start, end], including both ends. + const start = offset; + const end = offset + limit - 1; + + if (app.has("state")) { + blocks = app.resolve("state").getLastBlocksByHeight(start, end); + } + + if (blocks.length !== limit) { + blocks = await this.connection.blocksRepository.heightRange(start, end); + + await this.loadTransactionsForBlocks(blocks); + } + + return blocks; + } + + public async getBlocksForRound(round?: number) { + let lastBlock; + if (app.has("state")) { + lastBlock = app.resolve("state").getLastBlock(); + } else { + lastBlock = await this.getLastBlock(); + } + + if (!lastBlock) { + return []; + } + + let height = +lastBlock.data.height; + if (!round) { + round = roundCalculator.calculateRound(height).round; + } + + const maxDelegates = this.config.getMilestone(height).activeDelegates; + height = round * maxDelegates + 1; + + const blocks = await this.getBlocks(height - maxDelegates, maxDelegates); + return blocks.map(b => new Block(b)); + } + + public async getForgedTransactionsIds(ids: string[]) { + if (!ids.length) { + return []; + } + + const txs = await this.connection.transactionsRepository.forged(ids); + return txs.map(tx => tx.id); + } + + public async getLastBlock() { + const block = await this.connection.blocksRepository.latest(); + + if (!block) { + return null; + } + + const transactions = await this.connection.transactionsRepository.latestByBlock(block.id); + + block.transactions = transactions.map(({ serialized }) => Transaction.deserialize(serialized.toString("hex"))); + + return new Block(block); + } + + public async getCommonBlocks(ids: string[]) { + const state = app.resolve("state"); + let commonBlocks = state.getCommonBlocks(ids); + if (commonBlocks.length < ids.length) { + commonBlocks = await this.connection.blocksRepository.common(ids); + } + + return commonBlocks; + } + + public async getRecentBlockIds() { + const state = app.resolve("state"); + let blocks = state + .getLastBlockIds() + .reverse() + .slice(0, 10); + + if (blocks.length < 10) { + blocks = await this.connection.blocksRepository.recent(10); + blocks = blocks.map(block => block.id); + } + + return blocks; + } + + public async getTopBlocks(count: any) { + const blocks = await this.connection.blocksRepository.top(count); + + await this.loadTransactionsForBlocks(blocks); + + return blocks; + } + + public async getTransaction(id: string) { + return this.connection.transactionsRepository.findById(id); + } + + public async loadBlocksFromCurrentRound() { + this.blocksInCurrentRound = await this.getBlocksForRound(); + } + + public async loadTransactionsForBlocks(blocks) { + if (!blocks.length) { + return; + } + + const ids = blocks.map(block => block.id); + + let transactions = await this.connection.transactionsRepository.latestByBlocks(ids); + transactions = transactions.map(tx => { + const data = Transaction.deserialize(tx.serialized.toString("hex")); + data.blockId = tx.blockId; + return data; + }); + + for (const block of blocks) { + if (block.numberOfTransactions > 0) { + block.transactions = transactions.filter(transaction => transaction.blockId === block.id); + } + } + } + + public async revertBlock(block: models.Block) { + await this.revertRound(block.data.height); + await this.walletManager.revertBlock(block); + + assert(this.blocksInCurrentRound.pop().data.id === block.data.id); + + this.emitter.emit("block.reverted", block.data); + } + + public async revertRound(height: number) { + const { round, nextRound, maxDelegates } = roundCalculator.calculateRound(height); + + if (nextRound === round + 1 && height >= maxDelegates) { + this.logger.info(`Back to previous round: ${round.toLocaleString()} :back:`); + + const delegates = await this.calcPreviousActiveDelegates(round); + this.forgingDelegates = await this.getActiveDelegates(height, delegates); + + await this.deleteRound(nextRound); + } + } + + public async saveBlock(block: models.Block) { + await this.connection.saveBlock(block); + } + + public async saveRound(activeDelegates: any[]) { + this.logger.info(`Saving round ${activeDelegates[0].round.toLocaleString()}`); + + await this.connection.roundsRepository.insert(activeDelegates); + + this.emitter.emit("round.created", activeDelegates); + } + + public async saveWallets(force: boolean) { + const wallets = this.walletManager + .allByPublicKey() + .filter(wallet => wallet.publicKey && (force || wallet.dirty)); + + // Remove dirty flags first to not save all dirty wallets in the exit handler + // when called during a force insert right after SPV. + this.walletManager.clear(); + + await this.connection.saveWallets(wallets, force); + + this.logger.info(`${wallets.length} modified ${pluralize("wallet", wallets.length)} committed to database`); + + this.emitter.emit("wallet.saved", wallets.length); + + // NOTE: commented out as more use cases to be taken care of + // this.walletManager.purgeEmptyNonDelegates() + } + + public updateDelegateStats(delegates: any[]): void { + if (!delegates || !this.blocksInCurrentRound) { + return; + } + + this.logger.debug("Updating delegate statistics"); + + try { + delegates.forEach(delegate => { + const producedBlocks = this.blocksInCurrentRound.filter( + blockGenerator => blockGenerator.data.generatorPublicKey === delegate.publicKey, + ); + const wallet = this.walletManager.findByPublicKey(delegate.publicKey); + + if (producedBlocks.length === 0) { + wallet.missedBlocks++; + this.logger.debug( + `Delegate ${wallet.username} (${wallet.publicKey}) just missed a block. Total: ${ + wallet.missedBlocks + }`, + ); + wallet.dirty = true; + this.emitter.emit("forger.missing", { + delegate: wallet, + }); + } + }); + } catch (error) { + this.logger.error(error.stack); + } + } + + public async verifyBlockchain(): Promise<{ valid: boolean; errors: any[] }> { + const errors = []; + + const lastBlock = await this.getLastBlock(); + + // Last block is available + if (!lastBlock) { + errors.push("Last block is not available"); + } else { + const numberOfBlocks = await this.connection.blocksRepository.count(); + + // Last block height equals the number of stored blocks + if (lastBlock.data.height !== +numberOfBlocks) { + errors.push( + `Last block height: ${lastBlock.data.height.toLocaleString()}, number of stored blocks: ${numberOfBlocks}`, + ); + } + } + + const blockStats = await this.connection.blocksRepository.statistics(); + const transactionStats = await this.connection.transactionsRepository.statistics(); + + // Number of stored transactions equals the sum of block.numberOfTransactions in the database + if (blockStats.numberOfTransactions !== transactionStats.count) { + errors.push( + `Number of transactions: ${transactionStats.count}, number of transactions included in blocks: ${ + blockStats.numberOfTransactions + }`, + ); + } + + // Sum of all tx fees equals the sum of block.totalFee + if (blockStats.totalFee !== transactionStats.totalFee) { + errors.push( + `Total transaction fees: ${transactionStats.totalFee}, total of block.totalFee : ${ + blockStats.totalFee + }`, + ); + } + + // Sum of all tx amount equals the sum of block.totalAmount + if (blockStats.totalAmount !== transactionStats.totalAmount) { + errors.push( + `Total transaction amounts: ${transactionStats.totalAmount}, total of block.totalAmount : ${ + blockStats.totalAmount + }`, + ); + } + + return { + valid: !errors.length, + errors, + }; + } + + public async verifyTransaction(transaction: models.Transaction) { + const senderId = arkCrypto.getAddress(transaction.data.senderPublicKey, this.config.get("network.pubKeyHash")); + + const sender = this.walletManager.findByAddress(senderId); // should exist + + if (!sender.publicKey) { + sender.publicKey = transaction.data.senderPublicKey; + this.walletManager.reindex(sender); + } + + const dbTransaction = await this.getTransaction(transaction.data.id); + + return sender.canApply(transaction.data, []) && !dbTransaction; + } + + private async calcPreviousActiveDelegates(round: number) { + // TODO: cache the blocks of the last X rounds + this.blocksInCurrentRound = await this.getBlocksForRound(round); + + // Create temp wallet manager from all delegates + const tempWalletManager = new WalletManager(); + tempWalletManager.index(cloneDeep(this.walletManager.allByUsername())); + + // Revert all blocks in reverse order + let height = 0; + for (let i = this.blocksInCurrentRound.length - 1; i >= 0; i--) { + tempWalletManager.revertBlock(this.blocksInCurrentRound[i]); + height = this.blocksInCurrentRound[i].data.height; + } + + // The first round has no active delegates + if (height === 1) { + return []; + } + + // Assert that the height is the beginning of a round. + const { maxDelegates } = roundCalculator.calculateRound(height); + assert(height > 1 && height % maxDelegates === 1); + + // Now retrieve the active delegate list from the temporary wallet manager. + return tempWalletManager.loadActiveDelegateList(maxDelegates, height); + } + + private emitTransactionEvents(transaction) { + this.emitter.emit("transaction.applied", transaction.data); + + if (transaction.type === TransactionTypes.DelegateRegistration) { + this.emitter.emit("delegate.registered", transaction.data); + } + + if (transaction.type === TransactionTypes.DelegateResignation) { + this.emitter.emit("delegate.resigned", transaction.data); + } + + if (transaction.type === TransactionTypes.Vote) { + const vote = transaction.asset.votes[0]; + + this.emitter.emit(vote.startsWith("+") ? "wallet.vote" : "wallet.unvote", { + delegate: vote, + transaction: transaction.data, + }); + } + } + + private registerListeners() { + + this.emitter.on("state:started", () => { + this.stateStarted = true; + }); + + this.emitter.on("wallet.created.cold", async coldWallet => { + try { + const wallet = await this.connection.walletsRepository.findByAddress(coldWallet.address); + + if (wallet) { + Object.keys(wallet).forEach(key => { + if (["balance"].indexOf(key) !== -1) { + return; + } + + coldWallet[key] = key !== "voteBalance" ? wallet[key] : new Bignum(wallet[key]); + }); + } + } catch (err) { + this.logger.error(err); + } + }); + + this.emitter.once("shutdown", async () => { + if (!this.spvFinished) { + // Prevent dirty wallets to be saved when SPV didn't finish + this.walletManager.clear(); + } + }); + } + +} diff --git a/packages/core-database/src/index.ts b/packages/core-database/src/index.ts index cdce672996..51340494aa 100644 --- a/packages/core-database/src/index.ts +++ b/packages/core-database/src/index.ts @@ -1,5 +1,5 @@ export * from "./manager"; -export * from "./interface"; +export * from "./database-service-factory"; export * from "./wallet-manager"; export * from "./repositories/delegates"; export * from "./repositories/wallets"; diff --git a/packages/core-database/src/interface.ts b/packages/core-database/src/interface.ts deleted file mode 100644 index d5364a8f89..0000000000 --- a/packages/core-database/src/interface.ts +++ /dev/null @@ -1,478 +0,0 @@ -import { app } from "@arkecosystem/core-container"; -import { EventEmitter, Logger } from "@arkecosystem/core-interfaces"; -import { roundCalculator } from "@arkecosystem/core-utils"; -import { constants, crypto, models } from "@arkecosystem/crypto"; -import assert from "assert"; -import cloneDeep from "lodash/cloneDeep"; -import { DelegatesRepository } from "./repositories/delegates"; -import { WalletsRepository } from "./repositories/wallets"; -import { WalletManager } from "./wallet-manager"; - -const { Block } = models; -const { TransactionTypes } = constants; - -export abstract class ConnectionInterface { - // TODO: Convert these to protected/private and provide the appropriate get/setters - public config = app.getConfig(); - public logger = app.resolvePlugin("logger"); - public emitter = app.resolvePlugin("event-emitter"); - public blocksInCurrentRound: any[] = null; - public stateStarted: boolean = false; - public restoredDatabaseIntegrity: boolean = false; - public walletManager: WalletManager = null; - public forgingDelegates: any[] = null; - public wallets: WalletsRepository = null; - public delegates: DelegatesRepository = null; - public queuedQueries: any[] = null; - - /** - * @constructor - * @param {Object} options - */ - protected constructor(public readonly options: any) { - this.__registerListeners(); - } - - public abstract async make(): Promise; - - /** - * Connect to a database. - * @return {void} - * @throws Error - */ - public abstract async connect(): Promise; - - /** - * Disconnect from a database. - * @return {void} - * @throws Error - */ - public abstract async disconnect(): Promise; - - /** - * Verify the blockchain stored on db is not corrupted making simple assertions: - * - Last block is available - * - Last block height equals the number of stored blocks - * - Number of stored transactions equals the sum of block.numberOfTransactions in the database - * - Sum of all tx fees equals the sum of block.totalFee - * - Sum of all tx amount equals the sum of block.totalAmount - * @return {Object} An object { valid, errors } with the result of the verification and the errors - */ - public abstract async verifyBlockchain(): Promise; - - /** - * Get the top 51 delegates. - * @param {Number} height - * @param {Array} delegates - * @return {Array} - * @throws Error - */ - public abstract async getActiveDelegates(height, delegates?): Promise; - - /** - * Load a list of wallets into memory. - * @param {Number} height - * @return {Boolean} success - * @throws Error - */ - public abstract async buildWallets(height): Promise; - - /** - * Commit wallets from the memory. - * @param {Boolean} force - * @return {void} - * @throws Error - */ - public abstract async saveWallets(force): Promise; - - /** - * Commit the given block. - * NOTE: to be used when node is in sync and committing newly received blocks - * @param {Block} block - * @return {void} - * @throws Error - */ - public abstract async saveBlock(block): Promise; - - /** - * Queue a query to save the given block. - * NOTE: Must call commitQueuedQueries() to save to database. - * NOTE: to use when rebuilding to decrease the number of database transactions, - * and commit blocks (save only every 1000s for instance) by calling commit - * @param {Block} block - * @return {void} - * @throws Error - */ - public abstract enqueueSaveBlock(block): void; - - /** - * Queue a query to delete the given block. - * See also enqueueSaveBlock - * @param {Block} block - * @return {void} - * @throws Error - */ - public abstract enqueueDeleteBlock(block): void; - - /** - * Queue a query to delete the round at given height. - * See also enqueueSaveBlock and enqueueDeleteBlock - * @param {Number} height - * @return {void} - * @throws Error - */ - public abstract enqueueDeleteRound(height): void; - - /** - * Commit all queued queries to the database. - * NOTE: to be used in combination with other enqueue-functions. - * @return {void} - * @throws Error - */ - public abstract async commitQueuedQueries(): Promise; - - /** - * Delete the given block. - * @param {Block} block - * @return {void} - * @throws Error - */ - public abstract async deleteBlock(block): Promise; - - /** - * Get a block. - * @param {Block} id - * @return {void} - * @throws Error - */ - public abstract async getBlock(id): Promise; - - /** - * Get last block. - * @return {void} - * @throws Error - */ - public abstract async getLastBlock(): Promise; - - /** - * Get blocks for the given offset and limit. - * @param {Number} offset - * @param {Number} limit - * @return {void} - * @throws Error - */ - public abstract async getBlocks(offset, limit): Promise; - - /** - * Get top count blocks ordered by height DESC. - * NOTE: Only used when trying to restore database integrity. - * The returned blocks may be unchained. - * @param {Number} count - * @return {void} - * @throws Error - */ - public abstract async getTopBlocks(count): Promise; - - /** - * Get recent block ids. - * @return {[]String} - */ - public abstract async getRecentBlockIds(): Promise; - - /** - * Store the given round. - * @param {Array} activeDelegates - * @return {void} - * @throws Error - */ - public abstract async saveRound(activeDelegates): Promise; - /** - * Delete the given round. - * @param {Number} round - * @return {void} - * @throws Error - */ - public abstract async deleteRound(round): Promise; - - /** - * Get a transaction. - * @param {Number} id - * @return {Promise} - */ - public abstract async getTransaction(id): Promise; - - /** - * Load blocks from current round into memory. - * @return {void]} - */ - public async loadBlocksFromCurrentRound() { - this.blocksInCurrentRound = await this.__getBlocksForRound(); - } - - /** - * Update delegate statistics in memory. - * NOTE: must be called before saving new round of delegates - * @param {Block} block - * @param {Array} delegates - * @return {void} - */ - public updateDelegateStats(height, delegates) { - if (!delegates || !this.blocksInCurrentRound) { - return; - } - - this.logger.debug("Updating delegate statistics"); - - try { - delegates.forEach(delegate => { - const producedBlocks = this.blocksInCurrentRound.filter( - blockGenerator => blockGenerator.data.generatorPublicKey === delegate.publicKey, - ); - const wallet = this.walletManager.findByPublicKey(delegate.publicKey); - - if (producedBlocks.length === 0) { - wallet.missedBlocks++; - this.logger.debug( - `Delegate ${wallet.username} (${wallet.publicKey}) just missed a block. Total: ${ - wallet.missedBlocks - }`, - ); - wallet.dirty = true; - this.emitter.emit("forger.missing", { - delegate: wallet, - }); - } - }); - } catch (error) { - this.logger.error(error.stack); - } - } - - /** - * Apply the round. - * Note that the round is applied and the end of the round (so checking height + 1) - * so the next block to apply starting the new round will be ready to be validated - * @param {Number} height - * @return {void} - */ - public async applyRound(height) { - const nextHeight = height === 1 ? 1 : height + 1; - const maxDelegates = this.config.getMilestone(nextHeight).activeDelegates; - - if (nextHeight % maxDelegates === 1) { - const round = Math.floor((nextHeight - 1) / maxDelegates) + 1; - - if ( - !this.forgingDelegates || - this.forgingDelegates.length === 0 || - (this.forgingDelegates.length && this.forgingDelegates[0].round !== round) - ) { - this.logger.info(`Starting Round ${round.toLocaleString()} :dove_of_peace:`); - - try { - this.updateDelegateStats(height, this.forgingDelegates); - this.saveWallets(false); // save only modified wallets during the last round - const delegates = this.walletManager.loadActiveDelegateList(maxDelegates, nextHeight); // get active delegate list from in-memory wallet manager - this.saveRound(delegates); // save next round delegate list non-blocking - this.forgingDelegates = await this.getActiveDelegates(nextHeight, delegates); // generate the new active delegates list - this.blocksInCurrentRound.length = 0; - } catch (error) { - // trying to leave database state has it was - await this.deleteRound(round); - throw error; - } - } else { - this.logger.warn( - // tslint:disable-next-line:max-line-length - `Round ${round.toLocaleString()} has already been applied. This should happen only if you are a forger. :warning:`, - ); - } - } - } - - /** - * Remove the round. - * @param {Number} height - * @return {void} - */ - public async revertRound(height) { - const { round, nextRound, maxDelegates } = roundCalculator.calculateRound(height); - - if (nextRound === round + 1 && height >= maxDelegates) { - this.logger.info(`Back to previous round: ${round.toLocaleString()} :back:`); - - const delegates = await this.__calcPreviousActiveDelegates(round); - this.forgingDelegates = await this.getActiveDelegates(height, delegates); - - await this.deleteRound(nextRound); - } - } - - /** - * Calculate the active delegates of the previous round. In order to do - * so we need to go back to the start of that round. Therefore we create - * a temporary wallet manager with all delegates and revert all blocks - * and transactions of that round to get the initial vote balances - * which are then used to restore the original order. - * @param {Number} round - */ - public async __calcPreviousActiveDelegates(round) { - // TODO: cache the blocks of the last X rounds - this.blocksInCurrentRound = await this.__getBlocksForRound(round); - - // Create temp wallet manager from all delegates - const tempWalletManager = new WalletManager(); - tempWalletManager.index(cloneDeep(this.walletManager.allByUsername())); - - // Revert all blocks in reverse order - let height = 0; - for (let i = this.blocksInCurrentRound.length - 1; i >= 0; i--) { - tempWalletManager.revertBlock(this.blocksInCurrentRound[i]); - height = this.blocksInCurrentRound[i].data.height; - } - - // The first round has no active delegates - if (height === 1) { - return []; - } - - // Assert that the height is the beginning of a round. - const { maxDelegates } = roundCalculator.calculateRound(height); - assert(height > 1 && height % maxDelegates === 1); - - // Now retrieve the active delegate list from the temporary wallet manager. - return tempWalletManager.loadActiveDelegateList(maxDelegates, height); - } - - /** - * Apply the given block. - */ - public async applyBlock(block: any): Promise { - this.walletManager.applyBlock(block); - - if (this.blocksInCurrentRound) { - this.blocksInCurrentRound.push(block); - } - - await this.applyRound(block.data.height); - block.transactions.forEach(tx => this.__emitTransactionEvents(tx)); - this.emitter.emit("block.applied", block.data); - return true; - } - - /** - * Remove the given block. - * @param {Block} block - * @return {void} - */ - public async revertBlock(block) { - await this.revertRound(block.data.height); - await this.walletManager.revertBlock(block); - - assert(this.blocksInCurrentRound.pop().data.id === block.data.id); - - this.emitter.emit("block.reverted", block.data); - } - - /** - * Verify a transaction. - * @param {Transaction} transaction - * @return {Boolean} - */ - public async verifyTransaction(transaction) { - const senderId = crypto.getAddress(transaction.data.senderPublicKey, this.config.get("network.pubKeyHash")); - - const sender = this.walletManager.findByAddress(senderId); // should exist - - if (!sender.publicKey) { - sender.publicKey = transaction.data.senderPublicKey; - this.walletManager.reindex(sender); - } - - const dbTransaction = await this.getTransaction(transaction.data.id); - - return sender.canApply(transaction.data, []) && !dbTransaction; - } - - /** - * Get blocks for round. - * @param {number} round - * @return {[]Block} - */ - public async __getBlocksForRound(round?) { - let lastBlock; - if (app.has("state")) { - lastBlock = app.resolve("state").getLastBlock(); - } else { - lastBlock = await this.getLastBlock(); - } - - if (!lastBlock) { - return []; - } - - let height = +lastBlock.data.height; - if (!round) { - round = roundCalculator.calculateRound(height).round; - } - - const maxDelegates = this.config.getMilestone(height).activeDelegates; - height = round * maxDelegates + 1; - - const blocks = await this.getBlocks(height - maxDelegates, maxDelegates); - return blocks.map(b => new Block(b)); - } - - /** - * Register event listeners. - * @return {void} - */ - public __registerListeners() { - this.emitter.on("state:started", () => { - this.stateStarted = true; - }); - } - - /** - * Register the wallet app. - * @return {void} - */ - public _registerWalletManager() { - this.walletManager = new WalletManager(); - } - - /** - * Register the wallet and delegate repositories. - * @return {void} - */ - public _registerRepositories() { - this.wallets = new WalletsRepository(this); - this.delegates = new DelegatesRepository(this); - } - - /** - * Emit events for the specified transaction. - * @param {Object} transaction - * @return {void} - */ - private __emitTransactionEvents(transaction) { - this.emitter.emit("transaction.applied", transaction.data); - - if (transaction.type === TransactionTypes.DelegateRegistration) { - this.emitter.emit("delegate.registered", transaction.data); - } - - if (transaction.type === TransactionTypes.DelegateResignation) { - this.emitter.emit("delegate.resigned", transaction.data); - } - - if (transaction.type === TransactionTypes.Vote) { - const vote = transaction.asset.votes[0]; - - this.emitter.emit(vote.startsWith("+") ? "wallet.vote" : "wallet.unvote", { - delegate: vote, - transaction: transaction.data, - }); - } - } -} diff --git a/packages/core-database/src/manager.ts b/packages/core-database/src/manager.ts index 3cf4676c2e..41721bcd62 100644 --- a/packages/core-database/src/manager.ts +++ b/packages/core-database/src/manager.ts @@ -1,7 +1,7 @@ -import { ConnectionInterface } from "./interface"; +import { Database } from "@arkecosystem/core-interfaces"; export class DatabaseManager { - public connections: { [key: string]: ConnectionInterface }; + public connections: { [key: string]: Database.IDatabaseConnection }; /** * Create a new database manager instance. @@ -14,19 +14,19 @@ export class DatabaseManager { /** * Get a database connection instance. * @param {String} name - * @return {ConnectionInterface} + * @return {DatabaseConnection} */ - public connection(name = "default"): ConnectionInterface { + public connection(name = "default"): Database.IDatabaseConnection { return this.connections[name]; } /** * Make the database connection instance. - * @param {ConnectionInterface} connection + * @param {DatabaseConnection} connection * @param {String} name * @return {void} */ - public async makeConnection(connection: ConnectionInterface, name = "default"): Promise { + public async makeConnection(connection: Database.IDatabaseConnection, name = "default"): Promise { this.connections[name] = await connection.make(); return this.connection(name); } diff --git a/packages/core-database/src/repositories/delegates.ts b/packages/core-database/src/repositories/delegates.ts index 372c071022..f273070024 100644 --- a/packages/core-database/src/repositories/delegates.ts +++ b/packages/core-database/src/repositories/delegates.ts @@ -1,20 +1,22 @@ +import { Database } from "@arkecosystem/core-interfaces"; import { delegateCalculator } from "@arkecosystem/core-utils"; import orderBy from "lodash/orderBy"; import limitRows from "./utils/limit-rows"; -export class DelegatesRepository { +export class DelegatesRepository implements Database.IDelegatesBusinessRepository { + /** * Create a new delegate repository instance. - * @param {ConnectionInterface} connection + * @param databaseServiceProvider */ - public constructor(public connection) {} + public constructor(private databaseServiceProvider : () => Database.IDatabaseService) {} /** * Get all local delegates. - * @return {Array} */ public getLocalDelegates() { - return this.connection.walletManager.all().filter(wallet => !!wallet.username); + // TODO: What's the diff between this and just calling 'allByUsername' + return this.databaseServiceProvider().walletManager.allByAddress().filter(wallet => !!wallet.username); } /** @@ -22,7 +24,7 @@ export class DelegatesRepository { * @param {Object} params * @return {Object} */ - public findAll(params: { orderBy?: string } = {}) { + public findAll(params: Database.IParameters = {}) { const delegates = this.getLocalDelegates(); const [iteratee, order] = this.__orderBy(params); @@ -33,26 +35,16 @@ export class DelegatesRepository { }; } - /** - * Paginate all delegates. - * @param {Object} params - * @return {Object} - */ - public paginate(params) { - return this.findAll(params); - } - /** * Search all delegates. * TODO Currently it searches by username only * @param {Object} [params] * @param {String} [params.username] - Search by username - * @return {Object} */ - public search(params) { + public search(params : Database.IParameters) { let delegates = this.getLocalDelegates(); if (params.hasOwnProperty("username")) { - delegates = delegates.filter(delegate => delegate.username.indexOf(params.username) > -1); + delegates = delegates.filter(delegate => delegate.username.indexOf(params.username as string) > -1); } if (params.orderBy) { @@ -92,11 +84,11 @@ export class DelegatesRepository { * @param {Number} height * @return {Array} */ - public getActiveAtHeight(height) { - const delegates = this.connection.getActiveDelegates(height); + public async getActiveAtHeight(height: number) { + const delegates = await this.databaseServiceProvider().getActiveDelegates(height); return delegates.map(delegate => { - const wallet = this.connection.wallets.findById(delegate.publicKey); + const wallet = this.databaseServiceProvider().wallets.findById(delegate.publicKey); return { username: wallet.username, diff --git a/packages/core-database/src/repositories/utils/filter-rows.ts b/packages/core-database/src/repositories/utils/filter-rows.ts index c5b1c59329..0cf7962b95 100644 --- a/packages/core-database/src/repositories/utils/filter-rows.ts +++ b/packages/core-database/src/repositories/utils/filter-rows.ts @@ -5,7 +5,7 @@ * @param {Object} filters * @return {Array} */ -export = (rows, params, filters) => +export = (rows: T[], params, filters) => rows.filter(item => { if (filters.hasOwnProperty("exact")) { for (const elem of filters.exact) { diff --git a/packages/core-database/src/repositories/utils/limit-rows.ts b/packages/core-database/src/repositories/utils/limit-rows.ts index 9d3f1628da..169521762c 100644 --- a/packages/core-database/src/repositories/utils/limit-rows.ts +++ b/packages/core-database/src/repositories/utils/limit-rows.ts @@ -1,10 +1,8 @@ +import { Database } from "@arkecosystem/core-interfaces"; /** * Return some rows by an offset and a limit. - * @param {Array} rows - * @param {Object} params - * @return {Array} */ -export = (rows, params) => { +export = (rows: T[], params: Database.IParameters) => { if (params.offset || params.limit) { const offset = params.offset || 0; const limit = params.limit ? offset + params.limit : rows.length; diff --git a/packages/core-database/src/repositories/wallets.ts b/packages/core-database/src/repositories/wallets.ts index 4b03a00b2b..1df7856c1c 100644 --- a/packages/core-database/src/repositories/wallets.ts +++ b/packages/core-database/src/repositories/wallets.ts @@ -1,21 +1,21 @@ -import { Bignum } from "@arkecosystem/crypto"; +import { Database } from "@arkecosystem/core-interfaces"; import orderBy from "lodash/orderBy"; import filterRows from "./utils/filter-rows"; import limitRows from "./utils/limit-rows"; -export class WalletsRepository { +export class WalletsRepository implements Database.IWalletsBusinessRepository { /** * Create a new wallet repository instance. - * @param {ConnectionInterface} connection + * @param {DatabaseConnection} databaseService */ - public constructor(public connection) {} + public constructor(private databaseServiceProvider : () => Database.IDatabaseService) {} /** * Get all local wallets. * @return {Array} */ public all() { - return this.connection.walletManager.all(); + return this.databaseServiceProvider().walletManager.allByAddress(); } /** @@ -23,7 +23,7 @@ export class WalletsRepository { * @param {{ orderBy?: string }} params * @return {Object} */ - public findAll(params: { orderBy?: string } = {}) { + public findAll(params: Database.IParameters = {}) { const wallets = this.all(); const [iteratee, order] = params.orderBy ? params.orderBy.split(":") : ["rate", "asc"]; @@ -40,7 +40,7 @@ export class WalletsRepository { * @param {Object} params * @return {Object} */ - public findAllByVote(publicKey, params = {}) { + public findAllByVote(publicKey: string, params: Database.IParameters = {}) { const wallets = this.all().filter(wallet => wallet.vote === publicKey); return { @@ -51,16 +51,13 @@ export class WalletsRepository { /** * Find a wallet by address, public key or username. - * @param {Number} id - * @return {Object} */ - public findById(id) { + public findById(id: string) { return this.all().find(wallet => wallet.address === id || wallet.publicKey === id || wallet.username === id); } /** * Count all wallets. - * @return {Number} */ public count() { return this.all().length; @@ -68,10 +65,8 @@ export class WalletsRepository { /** * Find all wallets sorted by balance. - * @param {Object} params - * @return {Object} */ - public top(params = {}) { + public top(params: Database.IParameters = {}) { const wallets = Object.values(this.all()).sort((a: any, b: any) => +b.balance.minus(a.balance).toFixed()); return { @@ -100,7 +95,7 @@ export class WalletsRepository { * @param {Number} [params.voteBalance.to] - Search by voteBalance (maximum) * @return {Object} */ - public search(params) { + public search(params: T) { const query: any = { exact: ["address", "publicKey", "secondPublicKey", "username", "vote"], between: ["balance", "voteBalance"], diff --git a/packages/core-database/src/wallet-manager.ts b/packages/core-database/src/wallet-manager.ts index 4547cdad50..886d5f1a4d 100644 --- a/packages/core-database/src/wallet-manager.ts +++ b/packages/core-database/src/wallet-manager.ts @@ -1,15 +1,15 @@ import { app } from "@arkecosystem/core-container"; -import { Logger } from "@arkecosystem/core-interfaces"; +import { Database, Logger } from "@arkecosystem/core-interfaces"; import { roundCalculator } from "@arkecosystem/core-utils"; -import { constants, crypto, formatArktoshi, isException, models } from "@arkecosystem/crypto"; +import { Bignum, constants, crypto, formatArktoshi, isException, models } from "@arkecosystem/crypto"; import pluralize from "pluralize"; const { Wallet } = models; const { TransactionTypes } = constants; -export class WalletManager { - public logger: Logger.ILogger; - public config: any; +export class WalletManager implements Database.IWalletManager { + public logger = app.resolvePlugin("logger"); + public config = app.getConfig(); public networkId: number; public byAddress: { [key: string]: any }; @@ -21,36 +21,18 @@ export class WalletManager { * @constructor */ constructor() { - this.config = app.getConfig(); - this.logger = app.resolvePlugin("logger"); - this.networkId = this.config ? this.config.get("network.pubKeyHash") : 0x17; this.reset(); } - /** - * Reset the wallets index. - * @return {void} - */ - public reset() { - this.byAddress = {}; - this.byPublicKey = {}; - this.byUsername = {}; - } - - /** - * Get all wallets by address. - * @return {Array} - */ - public all() { + public allByAddress(): models.Wallet[] { return Object.values(this.byAddress); } /** * Get all wallets by publicKey. - * @return {Array} */ - public allByPublicKey() { + public allByPublicKey(): models.Wallet[] { return Object.values(this.byPublicKey); } @@ -58,16 +40,14 @@ export class WalletManager { * Get all wallets by username. * @return {Array} */ - public allByUsername() { + public allByUsername(): models.Wallet[] { return Object.values(this.byUsername); } /** * Find a wallet by the given address. - * @param {String} address - * @return {Wallet} */ - public findByAddress(address) { + public findByAddress(address: string): models.Wallet { if (!this.byAddress[address]) { this.byAddress[address] = new Wallet(address); } @@ -78,17 +58,13 @@ export class WalletManager { /** * Checks if wallet exits in wallet manager * @param {String} key can be publicKey or address of wallet - * @return {Boolean} true if exists */ - public exists(key) { + public exists(key: string) { if (this.byPublicKey[key]) { return true; } - if (this.byAddress[key]) { - return true; - } - return false; + return !!this.byAddress[key]; } /** @@ -96,7 +72,7 @@ export class WalletManager { * @param {String} publicKey * @return {Wallet} */ - public findByPublicKey(publicKey) { + public findByPublicKey(publicKey: string): models.Wallet { if (!this.byPublicKey[publicKey]) { const address = crypto.getAddress(publicKey, this.networkId); @@ -113,7 +89,7 @@ export class WalletManager { * @param {String} username * @return {Wallet} */ - public findByUsername(username) { + public findByUsername(username: string): models.Wallet { return this.byUsername[username]; } @@ -121,7 +97,6 @@ export class WalletManager { * Set wallet by address. * @param {String} address * @param {Wallet} wallet - * @param {void} */ public setByAddress(address, wallet) { this.byAddress[address] = wallet; @@ -131,7 +106,6 @@ export class WalletManager { * Set wallet by publicKey. * @param {String} publicKey * @param {Wallet} wallet - * @param {void} */ public setByPublicKey(publicKey, wallet) { this.byPublicKey[publicKey] = wallet; @@ -141,7 +115,6 @@ export class WalletManager { * Set wallet by username. * @param {String} username * @param {Wallet} wallet - * @param {void} */ public setByUsername(username, wallet) { this.byUsername[username] = wallet; @@ -150,7 +123,6 @@ export class WalletManager { /** * Remove wallet by address. * @param {String} address - * @param {void} */ public forgetByAddress(address) { delete this.byAddress[address]; @@ -159,7 +131,6 @@ export class WalletManager { /** * Remove wallet by publicKey. * @param {String} publicKey - * @param {void} */ public forgetByPublicKey(publicKey) { delete this.byPublicKey[publicKey]; @@ -168,7 +139,6 @@ export class WalletManager { /** * Remove wallet by username. * @param {String} username - * @param {void} */ public forgetByUsername(username) { delete this.byUsername[username]; @@ -190,7 +160,7 @@ export class WalletManager { * @param {Wallet} wallet * @return {void} */ - public reindex(wallet) { + public reindex(wallet: models.Wallet) { if (wallet.address) { this.byAddress[wallet.address] = wallet; } @@ -213,27 +183,28 @@ export class WalletManager { /** * Load a list of all active delegates. * @param {Number} maxDelegates + * @param height * @return {Array} */ - public loadActiveDelegateList(maxDelegates, height) { + public loadActiveDelegateList(maxDelegates: number, height?: number): any[] { if (height > 1 && height % maxDelegates !== 1) { throw new Error("Trying to build delegates outside of round change"); } const { round } = roundCalculator.calculateRound(height, maxDelegates); - let delegates = this.allByUsername(); + const delegatesWallets = this.allByUsername(); - if (delegates.length < maxDelegates) { + if (delegatesWallets.length < maxDelegates) { throw new Error( `Expected to find ${maxDelegates} delegates but only found ${ - delegates.length - }. This indicates an issue with the genesis block & delegates.`, + delegatesWallets.length + }. This indicates an issue with the genesis block & delegates.`, ); } const equalVotesMap = new Map(); - delegates = delegates + const delegates = delegatesWallets .sort((a, b) => { const diff = b.voteBalance.comparedTo(a.voteBalance); @@ -250,7 +221,7 @@ export class WalletManager { throw new Error( `The balance and public key of both delegates are identical! Delegate "${ a.username - }" appears twice in the list.`, + }" appears twice in the list.`, ); } @@ -303,7 +274,7 @@ export class WalletManager { */ public purgeEmptyNonDelegates() { Object.values(this.byPublicKey).forEach(wallet => { - if (this.__canBePurged(wallet)) { + if (this.canBePurged(wallet)) { delete this.byPublicKey[wallet.publicKey]; delete this.byAddress[wallet.address]; } @@ -315,7 +286,7 @@ export class WalletManager { * @param {Block} block * @return {void} */ - public applyBlock(block) { + public applyBlock(block: models.Block) { const generatorPublicKey = block.data.generatorPublicKey; let delegate = this.byPublicKey[block.data.generatorPublicKey]; @@ -353,7 +324,7 @@ export class WalletManager { // by reward + totalFee. In which case the vote balance of the // delegate's delegate has to be updated. if (applied && delegate.vote) { - const increase = block.data.reward.plus(block.data.totalFee); + const increase = (block.data.reward as Bignum).plus(block.data.totalFee); const votedDelegate = this.byPublicKey[delegate.vote]; votedDelegate.voteBalance = votedDelegate.voteBalance.plus(increase); } @@ -376,7 +347,7 @@ export class WalletManager { * @param {Block} block * @return {void} */ - public async revertBlock(block) { + public revertBlock(block: models.Block) { const delegate = this.byPublicKey[block.data.generatorPublicKey]; if (!delegate) { @@ -401,7 +372,7 @@ export class WalletManager { // by reward + totalFee. In which case the vote balance of the // delegate's delegate has to be updated. if (reverted && delegate.vote) { - const decrease = block.data.reward.plus(block.data.totalFee); + const decrease = (block.data.reward as Bignum).plus(block.data.totalFee); const votedDelegate = this.byPublicKey[delegate.vote]; votedDelegate.voteBalance = votedDelegate.voteBalance.minus(decrease); } @@ -419,7 +390,7 @@ export class WalletManager { * @param {Transaction} transaction * @return {Transaction} */ - public applyTransaction(transaction) { + public applyTransaction(transaction: models.Transaction) { const { data } = transaction; const { type, asset, recipientId, senderPublicKey } = data; @@ -432,13 +403,13 @@ export class WalletManager { this.logger.error( `Can't apply transaction ${ data.id - }: delegate name '${asset.delegate.username.toLowerCase()}' already taken.`, + }: delegate name '${asset.delegate.username.toLowerCase()}' already taken.`, ); throw new Error(`Can't apply transaction ${data.id}: delegate name already taken.`); // NOTE: We use the vote public key, because vote transactions // have the same sender and recipient - } else if (type === TransactionTypes.Vote && !this.__isDelegate(asset.votes[0].slice(1))) { + } else if (type === TransactionTypes.Vote && !this.isDelegate(asset.votes[0].slice(1))) { this.logger.error(`Can't apply vote transaction ${data.id}: delegate ${asset.votes[0]} does not exist.`); throw new Error(`Can't apply transaction ${data.id}: delegate ${asset.votes[0]} does not exist.`); } else if (type === TransactionTypes.SecondSignature) { @@ -525,7 +496,7 @@ export class WalletManager { * @param {Transaction} transaction * @return {Transaction} */ - public revertTransaction(transaction) { + public revertTransaction(transaction: models.Transaction) { const { type, data } = transaction; const sender = this.findByPublicKey(data.senderPublicKey); // Should exist const recipient = this.byAddress[data.recipientId]; @@ -551,7 +522,7 @@ export class WalletManager { * Checks if a given publicKey is a registered delegate * @param {String} publicKey */ - public __isDelegate(publicKey) { + public isDelegate(publicKey: string) { const delegateWallet = this.byPublicKey[publicKey]; if (delegateWallet && delegateWallet.username) { @@ -566,7 +537,17 @@ export class WalletManager { * @param {Object} wallet * @return {Boolean} */ - public __canBePurged(wallet) { + public canBePurged(wallet) { return wallet.balance.isZero() && !wallet.secondPublicKey && !wallet.multisignature && !wallet.username; } + + /** + * Reset the wallets index. + * @return {void} + */ + public reset() { + this.byAddress = {}; + this.byPublicKey = {}; + this.byUsername = {}; + } } diff --git a/packages/core-elasticsearch/package.json b/packages/core-elasticsearch/package.json index 5ba4c03824..fc485a5fe2 100644 --- a/packages/core-elasticsearch/package.json +++ b/packages/core-elasticsearch/package.json @@ -24,7 +24,6 @@ "dependencies": { "@arkecosystem/core-interfaces": "^2.1.0", "@arkecosystem/core-container": "^2.1.0", - "@arkecosystem/core-database-postgres": "^2.1.0", "@arkecosystem/core-http-utils": "^2.1.0", "@arkecosystem/crypto": "^2.1.0", "@types/elasticsearch": "^5.0.30", diff --git a/packages/core-elasticsearch/src/index/block.ts b/packages/core-elasticsearch/src/index/block.ts index ffacee856f..c1516d02a1 100644 --- a/packages/core-elasticsearch/src/index/block.ts +++ b/packages/core-elasticsearch/src/index/block.ts @@ -1,15 +1,13 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { EventEmitter, Logger } from "@arkecosystem/core-interfaces"; +import { Database, Logger } from "@arkecosystem/core-interfaces"; import first from "lodash/first"; import last from "lodash/last"; import { client } from "../services/client"; import { storage } from "../services/storage"; import { Index } from "./index"; -const emitter = app.resolvePlugin("event-emitter"); const logger = app.resolvePlugin("logger"); -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); class BlockIndex extends Index { /** @@ -32,7 +30,7 @@ class BlockIndex extends Index { .limit(this.chunkSize) .offset(this.chunkSize * i); - const rows = await database.query.manyOrNone(query.toQuery()); + const rows = await (databaseService.connection as any).query.manyOrNone(query.toQuery()); if (!rows.length) { continue; diff --git a/packages/core-elasticsearch/src/index/index.ts b/packages/core-elasticsearch/src/index/index.ts index 61513f3ebc..d2b7fc9f04 100644 --- a/packages/core-elasticsearch/src/index/index.ts +++ b/packages/core-elasticsearch/src/index/index.ts @@ -1,12 +1,11 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { EventEmitter, Logger } from "@arkecosystem/core-interfaces"; +import { Database, EventEmitter, Logger } from "@arkecosystem/core-interfaces"; import { client } from "../services/client"; import { storage } from "../services/storage"; const emitter = app.resolvePlugin("event-emitter"); const logger = app.resolvePlugin("logger"); -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); export abstract class Index { public chunkSize: any; @@ -173,7 +172,7 @@ export abstract class Index { } public __createQuery() { - return database.models[this.getType()].query(); + return (databaseService.connection as any).models[this.getType()].query(); } public __count() { @@ -181,6 +180,6 @@ export abstract class Index { const query = modelQuery.select(modelQuery.count("count")).from(modelQuery); - return database.query.one(query.toQuery()); + return (databaseService.connection as any).query.one(query.toQuery()); } } diff --git a/packages/core-elasticsearch/src/index/round.ts b/packages/core-elasticsearch/src/index/round.ts index 95d3b740a8..cae825e467 100644 --- a/packages/core-elasticsearch/src/index/round.ts +++ b/packages/core-elasticsearch/src/index/round.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { EventEmitter, Logger } from "@arkecosystem/core-interfaces"; +import { Database, EventEmitter, Logger } from "@arkecosystem/core-interfaces"; import first from "lodash/first"; import last from "lodash/last"; import { client } from "../services/client"; @@ -9,7 +8,7 @@ import { Index } from "./index"; const emitter = app.resolvePlugin("event-emitter"); const logger = app.resolvePlugin("logger"); -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); class RoundIndex extends Index { /** @@ -32,7 +31,7 @@ class RoundIndex extends Index { .limit(this.chunkSize) .offset(this.chunkSize * i); - const rows = await database.query.manyOrNone(query.toQuery()); + const rows = await (databaseService.connection as any).query.manyOrNone(query.toQuery()); if (!rows.length) { continue; diff --git a/packages/core-elasticsearch/src/index/transaction.ts b/packages/core-elasticsearch/src/index/transaction.ts index 88e92969fa..04fb7e4677 100644 --- a/packages/core-elasticsearch/src/index/transaction.ts +++ b/packages/core-elasticsearch/src/index/transaction.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { EventEmitter, Logger } from "@arkecosystem/core-interfaces"; +import { Database, EventEmitter, Logger } from "@arkecosystem/core-interfaces"; import first from "lodash/first"; import last from "lodash/last"; import { client } from "../services/client"; @@ -10,9 +9,8 @@ import { Index } from "./index"; import { models } from "@arkecosystem/crypto"; const { Transaction } = models; -const emitter = app.resolvePlugin("event-emitter"); const logger = app.resolvePlugin("logger"); -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); class TransactionIndex extends Index { /** @@ -35,7 +33,7 @@ class TransactionIndex extends Index { .limit(this.chunkSize) .offset(this.chunkSize * i); - let rows = await database.query.manyOrNone(query.toQuery()); + let rows = await (databaseService.connection as any).query.manyOrNone(query.toQuery()); if (!rows.length) { continue; diff --git a/packages/core-elasticsearch/src/index/wallet.ts b/packages/core-elasticsearch/src/index/wallet.ts index 1e2b1eb23a..2707be2d2b 100644 --- a/packages/core-elasticsearch/src/index/wallet.ts +++ b/packages/core-elasticsearch/src/index/wallet.ts @@ -1,12 +1,11 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { EventEmitter, Logger } from "@arkecosystem/core-interfaces"; +import { Database, EventEmitter, Logger } from "@arkecosystem/core-interfaces"; import { client } from "../services/client"; import { Index } from "./index"; const emitter = app.resolvePlugin("event-emitter"); const logger = app.resolvePlugin("logger"); -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); class WalletIndex extends Index { /** @@ -27,7 +26,7 @@ class WalletIndex extends Index { .limit(this.chunkSize) .offset(this.chunkSize * i); - const rows = await database.query.manyOrNone(query.toQuery()); + const rows = await (databaseService.connection as any).query.manyOrNone(query.toQuery()); if (!rows.length) { continue; diff --git a/packages/core-graphql/__tests__/api/transaction.test.ts b/packages/core-graphql/__tests__/api/transaction.test.ts index 14062b3976..131d8f6150 100644 --- a/packages/core-graphql/__tests__/api/transaction.test.ts +++ b/packages/core-graphql/__tests__/api/transaction.test.ts @@ -8,8 +8,8 @@ beforeAll(async () => { await setUp(); }); -afterAll(() => { - tearDown(); +afterAll( async () => { + await tearDown(); }); describe("GraphQL API { transaction }", () => { @@ -18,7 +18,7 @@ describe("GraphQL API { transaction }", () => { const query = `{ transaction(id:"${genesisBlock.transactions[0].id}") { id } }`; const response = await utils.request(query); - expect(response).toBeSuccessfulResponse(); + await expect(response).toBeSuccessfulResponse(); const data = response.data.data; expect(data).toBeObject(); diff --git a/packages/core-graphql/package.json b/packages/core-graphql/package.json index 98ddea360e..4d04edd91f 100644 --- a/packages/core-graphql/package.json +++ b/packages/core-graphql/package.json @@ -29,7 +29,6 @@ "dependencies": { "@arkecosystem/core-interfaces": "^2.1.0", "@arkecosystem/core-container": "^2.1.0", - "@arkecosystem/core-database-postgres": "^2.1.0", "@arkecosystem/core-http-utils": "^2.1.0", "@arkecosystem/crypto": "^2.1.0", "apollo-server-hapi": "^2.3.1", diff --git a/packages/core-graphql/src/repositories/blocks.ts b/packages/core-graphql/src/repositories/blocks.ts index ed235dd5e8..f1e12c66a2 100644 --- a/packages/core-graphql/src/repositories/blocks.ts +++ b/packages/core-graphql/src/repositories/blocks.ts @@ -122,7 +122,7 @@ class BlocksRepository extends Repository { } public getModel() { - return this.database.models.block; + return (this.databaseService.connection as any).models.block; } public __orderBy(parameters) { diff --git a/packages/core-graphql/src/repositories/repository.ts b/packages/core-graphql/src/repositories/repository.ts index c231cb6000..21499a3fd0 100644 --- a/packages/core-graphql/src/repositories/repository.ts +++ b/packages/core-graphql/src/repositories/repository.ts @@ -1,22 +1,21 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { TransactionPool } from "@arkecosystem/core-interfaces"; +import { Database, TransactionPool } from "@arkecosystem/core-interfaces"; export abstract class Repository { - public database = app.resolvePlugin("database"); + public databaseService = app.resolvePlugin("database"); public transactionPool = app.resolvePlugin("transactionPool"); - public cache = this.database.getCache(); + public cache = this.databaseService.cache; public model = this.getModel(); public query = this.model.query(); public abstract getModel(): any; public async _find(query) { - return this.database.query.oneOrNone(query.toQuery()); + return (this.databaseService.connection as any).query.oneOrNone(query.toQuery()); } public async _findMany(query) { - return this.database.query.manyOrNone(query.toQuery()); + return (this.databaseService.connection as any).query.manyOrNone(query.toQuery()); } public async _findManyWithCount(selectQuery, countQuery, { limit, offset, orderBy }) { diff --git a/packages/core-graphql/src/repositories/transactions.ts b/packages/core-graphql/src/repositories/transactions.ts index a5ec560043..602eeb0c3f 100644 --- a/packages/core-graphql/src/repositories/transactions.ts +++ b/packages/core-graphql/src/repositories/transactions.ts @@ -329,7 +329,7 @@ class TransactionsRepository extends Repository { } public getModel() { - return this.database.models.transaction; + return (this.databaseService.connection as any).models.transaction; } /** @@ -338,7 +338,7 @@ class TransactionsRepository extends Repository { * @return {Object} */ public async __mapBlocksToTransactions(data) { - const blockQuery = this.database.models.block.query(); + const blockQuery = (this.databaseService.connection as any).models.block.query(); // Array... if (Array.isArray(data)) { @@ -428,7 +428,7 @@ class TransactionsRepository extends Repository { * @return {String} */ public __publicKeyFromSenderId(senderId) { - return this.database.walletManager.findByAddress(senderId).publicKey; + return this.databaseService.walletManager.findByAddress(senderId).publicKey; } public __orderBy(parameters) { diff --git a/packages/core-graphql/src/resolvers/queries/block/block.ts b/packages/core-graphql/src/resolvers/queries/block/block.ts index a198962eed..d6e1fcbc0f 100644 --- a/packages/core-graphql/src/resolvers/queries/block/block.ts +++ b/packages/core-graphql/src/resolvers/queries/block/block.ts @@ -1,9 +1,10 @@ import { app } from "@arkecosystem/core-container"; +import { Database } from "@arkecosystem/core-interfaces"; /** * Get a single block from the database * @return {Block} */ export async function block(_, { id }) { - return app.resolvePlugin("database").db.blocks.findById(id); + return app.resolvePlugin("database").connection.blocksRepository.findById(id); } diff --git a/packages/core-graphql/src/resolvers/queries/transaction/transaction.ts b/packages/core-graphql/src/resolvers/queries/transaction/transaction.ts index efb9f7aa16..e2c3b8d8c3 100644 --- a/packages/core-graphql/src/resolvers/queries/transaction/transaction.ts +++ b/packages/core-graphql/src/resolvers/queries/transaction/transaction.ts @@ -1,9 +1,10 @@ import { app } from "@arkecosystem/core-container"; +import { Database } from "@arkecosystem/core-interfaces"; /** * Get a single transaction from the database * @return {Transaction} */ export async function transaction(_, { id }) { - return app.resolvePlugin("database").db.transactions.findById(id); + return app.resolvePlugin("database").connection.transactionsRepository.findById(id); } diff --git a/packages/core-graphql/src/resolvers/queries/wallet/wallet.ts b/packages/core-graphql/src/resolvers/queries/wallet/wallet.ts index f8906c94b4..81feab2dd7 100644 --- a/packages/core-graphql/src/resolvers/queries/wallet/wallet.ts +++ b/packages/core-graphql/src/resolvers/queries/wallet/wallet.ts @@ -1,6 +1,7 @@ import { app } from "@arkecosystem/core-container"; +import { Database } from "@arkecosystem/core-interfaces"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); /** * Get a single wallet from the database @@ -8,5 +9,5 @@ const database = app.resolvePlugin("database"); */ export async function wallet(_, args: any) { const param = args.address || args.publicKey || args.username; - return database.wallets.findById(param); + return databaseService.wallets.findById(param); } diff --git a/packages/core-graphql/src/resolvers/queries/wallet/wallets.ts b/packages/core-graphql/src/resolvers/queries/wallet/wallets.ts index 98bec068da..0ae2f10353 100644 --- a/packages/core-graphql/src/resolvers/queries/wallet/wallets.ts +++ b/packages/core-graphql/src/resolvers/queries/wallet/wallets.ts @@ -1,7 +1,8 @@ import { app } from "@arkecosystem/core-container"; +import { Database } from "@arkecosystem/core-interfaces"; import { formatOrderBy } from "../../../helpers"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); /** * Get multiple wallets from the database @@ -13,11 +14,11 @@ export async function wallets(_, args: any) { const order = formatOrderBy(orderBy, "height:desc"); const result = filter && filter.vote - ? await database.wallets.findAllByVote(filter.vote, { + ? await databaseService.wallets.findAllByVote(filter.vote, { orderBy: order, ...params, }) - : await database.wallets.findAll({ orderBy: order, ...params }); + : await databaseService.wallets.findAll({ orderBy: order, ...params }); return result ? result.rows : []; } diff --git a/packages/core-graphql/src/resolvers/relationship/block.ts b/packages/core-graphql/src/resolvers/relationship/block.ts index 587c59cb44..7457dfd2ba 100644 --- a/packages/core-graphql/src/resolvers/relationship/block.ts +++ b/packages/core-graphql/src/resolvers/relationship/block.ts @@ -1,7 +1,8 @@ import { app } from "@arkecosystem/core-container"; +import { Database } from "@arkecosystem/core-interfaces"; import { formatOrderBy, unserializeTransactions } from "../../helpers"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); /** * Useful and common database operations with block data. @@ -16,14 +17,16 @@ export const Block = { async transactions(block, args) { const { orderBy, filter, ...params } = args; - const result = await database.transactions.findAll( + /* .findAll() method never existed on the TransactionRepository in core-database-postgres. This code would've blown chunks + const result = await database.connection.transactionsRepository.findAll( { ...filter, orderBy: formatOrderBy(orderBy, "timestamp:DESC"), ...params, }, false, - ); + );*/ + const result = null; const rows = result ? result.rows : []; return unserializeTransactions(rows); @@ -35,6 +38,6 @@ export const Block = { * @return {Wallet} */ generator(block) { - return database.wallets.findById(block.generatorPublicKey); + return databaseService.wallets.findById(block.generatorPublicKey); }, }; diff --git a/packages/core-graphql/src/resolvers/relationship/transaction.ts b/packages/core-graphql/src/resolvers/relationship/transaction.ts index a914b95f86..5999c67b99 100644 --- a/packages/core-graphql/src/resolvers/relationship/transaction.ts +++ b/packages/core-graphql/src/resolvers/relationship/transaction.ts @@ -1,6 +1,7 @@ import { app } from "@arkecosystem/core-container"; +import { Database } from "@arkecosystem/core-interfaces"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); /** * Useful and common database operations with transaction data. @@ -11,19 +12,19 @@ export const Transaction = { * @param {Transaction} transaction * @return {Block} */ - block: transaction => database.blocks.findById(transaction.blockId), + block: transaction => databaseService.connection.blocksRepository.findById(transaction.blockId), /** * Get the recipient of a transaction * @param {Transaction} transaction * @return {Wallet} */ - recipient: transaction => (transaction.recipientId ? database.wallets.findById(transaction.recipientId) : []), + recipient: transaction => (transaction.recipientId ? databaseService.wallets.findById(transaction.recipientId) : []), /** * Get the sender of a transaction * @param {Transaction} transaction * @return {Wallet} */ - sender: transaction => (transaction.senderPublicKey ? database.wallets.findById(transaction.senderPublicKey) : []), + sender: transaction => (transaction.senderPublicKey ? databaseService.wallets.findById(transaction.senderPublicKey) : []), }; diff --git a/packages/core-graphql/src/resolvers/relationship/wallet.ts b/packages/core-graphql/src/resolvers/relationship/wallet.ts index dee8d776d2..b0216a4722 100644 --- a/packages/core-graphql/src/resolvers/relationship/wallet.ts +++ b/packages/core-graphql/src/resolvers/relationship/wallet.ts @@ -1,7 +1,8 @@ import { app } from "@arkecosystem/core-container"; +import { Database } from "@arkecosystem/core-interfaces"; import { formatOrderBy, unserializeTransactions } from "../../helpers"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); /** * Useful and common database operations with wallet data. @@ -16,7 +17,7 @@ export const Wallet = { async transactions(wallet, args) { const { orderBy, filter, ...params } = args; - const walletOr = database.createCondition("OR", [ + const walletOr = (databaseService.connection as any).createCondition("OR", [ { senderPublicKey: wallet.publicKey, }, @@ -25,7 +26,8 @@ export const Wallet = { }, ]); - const result = await database.transactions.findAll( + /* TODO .findAll() method never existed on the TransactionRepository in core-database-postgres. This code would've blown chunks + const result = await databaseService.connection.transactionsRepository.findAll( { ...filter, orderBy: formatOrderBy(orderBy, "timestamp:DESC"), @@ -33,7 +35,8 @@ export const Wallet = { ...params, }, false, - ); + );*/ + const result = null; const rows = result ? result.rows : []; return unserializeTransactions(rows); @@ -50,13 +53,16 @@ export const Wallet = { params.generatorPublickKey = wallet.publicKey; - const result = database.blocks.findAll( + + /* TODO: .findAll() method never existed on the TransactionRepository in core-database-postgres. This code would've blown chunks + const result = databaseService.connection.blocksRepository.findAll( { orderBy: formatOrderBy(orderBy, "height:DESC"), ...params, }, false, - ); + );*/ + const result = null; const rows = result ? result.rows : []; return rows; }, diff --git a/packages/core-interfaces/src/core-database/business-repository/delegates-business-repository.ts b/packages/core-interfaces/src/core-database/business-repository/delegates-business-repository.ts new file mode 100644 index 0000000000..48e65c523c --- /dev/null +++ b/packages/core-interfaces/src/core-database/business-repository/delegates-business-repository.ts @@ -0,0 +1,15 @@ +import { models } from "@arkecosystem/crypto"; +import { IParameters } from "./parameters"; + +export interface IDelegatesBusinessRepository { + + getLocalDelegates(): models.Wallet[]; + + findAll(params?: IParameters): { count: number, rows: models.Wallet[] } + + search(params: T): { count: number, rows: models.Wallet[] } + + findById(id: string): models.Wallet; + + getActiveAtHeight(height: number): Promise> +} diff --git a/packages/core-interfaces/src/core-database/business-repository/index.ts b/packages/core-interfaces/src/core-database/business-repository/index.ts new file mode 100644 index 0000000000..a30abaafdc --- /dev/null +++ b/packages/core-interfaces/src/core-database/business-repository/index.ts @@ -0,0 +1,3 @@ +export * from "./wallets-business-repository"; +export * from "./delegates-business-repository" +export * from "./parameters"; diff --git a/packages/core-interfaces/src/core-database/business-repository/parameters.ts b/packages/core-interfaces/src/core-database/business-repository/parameters.ts new file mode 100644 index 0000000000..27df1df680 --- /dev/null +++ b/packages/core-interfaces/src/core-database/business-repository/parameters.ts @@ -0,0 +1,6 @@ +export interface IParameters { + offset?: number; + limit?: number; + orderBy?: string, + [key: string]: object | number | string | boolean +} diff --git a/packages/core-interfaces/src/core-database/business-repository/wallets-business-repository.ts b/packages/core-interfaces/src/core-database/business-repository/wallets-business-repository.ts new file mode 100644 index 0000000000..ad5f6f05e8 --- /dev/null +++ b/packages/core-interfaces/src/core-database/business-repository/wallets-business-repository.ts @@ -0,0 +1,20 @@ +import { models } from "@arkecosystem/crypto"; +import { IParameters } from "./parameters"; + +export interface IWalletsBusinessRepository { + + all(): models.Wallet[]; + + findAll(params?: IParameters): { count: number, rows: models.Wallet[] } + + findAllByVote(publicKey: string, params?: IParameters): { count: number, rows: models.Wallet[] }; + + findById(id: string): models.Wallet; + + count(): number; + + top(params?: IParameters): { count: number, rows: models.Wallet[] } + + search(params: T): { count: number, rows: models.Wallet[] } + +} diff --git a/packages/core-interfaces/src/core-database/database-connection.ts b/packages/core-interfaces/src/core-database/database-connection.ts new file mode 100644 index 0000000000..7ce5688307 --- /dev/null +++ b/packages/core-interfaces/src/core-database/database-connection.ts @@ -0,0 +1,41 @@ +import { IBlocksRepository } from "./database-repository"; +import { IRoundsRepository } from "./database-repository"; +import { ITransactionsRepository } from "./database-repository"; +import { IWalletsRepository } from "./database-repository"; + +import { models } from "@arkecosystem/crypto"; + +export interface IDatabaseConnection { + + options: any; + + blocksRepository : IBlocksRepository; + walletsRepository: IWalletsRepository; + roundsRepository: IRoundsRepository; + transactionsRepository: ITransactionsRepository; + + make(): Promise; + + connect(): Promise; + + disconnect(): Promise; + + buildWallets(height: number) : Promise; + + /* We have these methods on the connection since they rely on transactions, which is a DB specific detail + Keep DB specifics away from the service layer + */ + saveWallets(wallets: any[], force?: boolean) : Promise; + + saveBlock(block: models.Block): Promise; + + deleteBlock(block: models.Block): Promise; + + enqueueDeleteBlock(block: models.Block); + + enqueueDeleteRound(height: number); + + enqueueSaveBlock(block: models.Block); + + commitQueuedQueries(); +} diff --git a/packages/core-interfaces/src/core-database/database-repository/blocks-repository.ts b/packages/core-interfaces/src/core-database/database-repository/blocks-repository.ts new file mode 100644 index 0000000000..a1e377b542 --- /dev/null +++ b/packages/core-interfaces/src/core-database/database-repository/blocks-repository.ts @@ -0,0 +1,56 @@ +import { Bignum } from "@arkecosystem/crypto"; +import { IRepository } from "./repository"; + +export interface IBlocksRepository extends IRepository { + + /** + * Find a block by its ID. + */ + findById(id: string): Promise; + + /** + * Count the number of records in the database. + */ + count(): Promise; + + /** + * Get all of the common blocks from the database. + */ + common(ids: string[]): Promise + + /** + * Get all of the blocks within the given height range and order them by height. + */ + heightRange(start: number, end: number): Promise; + + /** + * Get the last created block from the database. + */ + latest(): Promise; + + /** + * Get the most recently created blocks ids from the database. + * @return {Promise} + */ + recent(count: number): Promise; + + /** + * Get statistics about all blocks from the database. + */ + statistics(): Promise<{ + numberOfTransactions: number, + totalFee: Bignum, + totalAmount: Bignum, + count: number + }>; + + /** + * Get top count blocks + */ + top(count: number): Promise; + + /** + * Delete the block from the database. + */ + delete(id: string): Promise; +} diff --git a/packages/core-interfaces/src/core-database/database-repository/index.ts b/packages/core-interfaces/src/core-database/database-repository/index.ts new file mode 100644 index 0000000000..104bbed884 --- /dev/null +++ b/packages/core-interfaces/src/core-database/database-repository/index.ts @@ -0,0 +1,5 @@ +export * from "./transactions-repository"; +export * from "./rounds-repository"; +export * from "./wallets-repository"; +export * from "./blocks-repository"; +export * from "./repository"; diff --git a/packages/core-interfaces/src/core-database/database-repository/repository.ts b/packages/core-interfaces/src/core-database/database-repository/repository.ts new file mode 100644 index 0000000000..20dbcfcc82 --- /dev/null +++ b/packages/core-interfaces/src/core-database/database-repository/repository.ts @@ -0,0 +1,11 @@ +export interface IRepository { + + estimate() : Promise; + + truncate(): Promise; + + insert(item: any | any[]) : Promise; + + update(item: any | any[]) : Promise; + +} diff --git a/packages/core-interfaces/src/core-database/database-repository/rounds-repository.ts b/packages/core-interfaces/src/core-database/database-repository/rounds-repository.ts new file mode 100644 index 0000000000..468ca091c1 --- /dev/null +++ b/packages/core-interfaces/src/core-database/database-repository/rounds-repository.ts @@ -0,0 +1,13 @@ +import { IRepository } from "./repository"; + +export interface IRoundsRepository extends IRepository { + /** + * Find a round by its ID. + */ + findById(id: number): Promise; + + /** + * Delete the round from the database. + */ + delete(id: number): Promise; +} diff --git a/packages/core-interfaces/src/core-database/database-repository/transactions-repository.ts b/packages/core-interfaces/src/core-database/database-repository/transactions-repository.ts new file mode 100644 index 0000000000..7df0d8c633 --- /dev/null +++ b/packages/core-interfaces/src/core-database/database-repository/transactions-repository.ts @@ -0,0 +1,45 @@ +import { Bignum } from "@arkecosystem/crypto"; +import { IRepository } from "./repository"; + +export interface ITransactionsRepository extends IRepository { + + /** + * Find a transactions by its ID. + */ + findById(id: string): Promise; + + /** + * Find multiple transactionss by their block ID. + */ + findByBlockId(blockId: string): Promise; + + /** + * Find multiple transactionss by their block ID and order them by sequence. + */ + latestByBlock(blockId: string): Promise; + + /** + * Find multiple transactionss by their block IDs and order them by sequence. + */ + latestByBlocks(blockIds: string[]): Promise; + + /** + * Get all of the forged transaction ids from the database. + */ + forged(ids: string[]): Promise; + + /** + * Get statistics about all transactions from the database. + */ + statistics(): Promise<{ + count: number, + totalFee: Bignum, + totalAmount: Bignum + }>; + + /** + * Delete transactions with blockId + */ + deleteByBlockId(blockId: string): Promise; + +} diff --git a/packages/core-interfaces/src/core-database/database-repository/wallets-repository.ts b/packages/core-interfaces/src/core-database/database-repository/wallets-repository.ts new file mode 100644 index 0000000000..b0046fa8b7 --- /dev/null +++ b/packages/core-interfaces/src/core-database/database-repository/wallets-repository.ts @@ -0,0 +1,28 @@ +import { IRepository } from "./repository"; + +export interface IWalletsRepository extends IRepository { + /** + * Get all of the wallets from the database. + */ + all(): Promise; + + /** + * Find a wallet by its address. + */ + findByAddress(address: string): Promise + + /** + * Get the count of wallets that have a negative balance. + */ + tallyWithNegativeBalance(): Promise; + + /** + * Get the count of wallets that have a negative vote balance. + */ + tallyWithNegativeVoteBalance(): Promise; + + /** + * Create or update a record matching the attributes, and fill it with values. + */ + updateOrCreate(wallet: any): Promise; +} diff --git a/packages/core-interfaces/src/core-database/database-service.ts b/packages/core-interfaces/src/core-database/database-service.ts new file mode 100644 index 0000000000..d290febd72 --- /dev/null +++ b/packages/core-interfaces/src/core-database/database-service.ts @@ -0,0 +1,90 @@ +import { models } from "@arkecosystem/crypto"; +import { EventEmitter, Logger } from "../index"; +import { IDelegatesBusinessRepository, IWalletsBusinessRepository } from "./business-repository"; +import { IDatabaseConnection } from "./database-connection"; +import { IWalletManager } from "./wallet-manager"; + +export interface IDatabaseService { + + walletManager: IWalletManager; + + wallets: IWalletsBusinessRepository; + + delegates: IDelegatesBusinessRepository; + + connection: IDatabaseConnection; + + logger: Logger.ILogger; + + emitter: EventEmitter.EventEmitter; + + config: any; + + options: any; + + cache: Map; + + restoredDatabaseIntegrity: boolean; + + verifyBlockchain(): Promise<{ valid: boolean, errors: any[] }>; + + getActiveDelegates(height: number, delegates?: any[]): Promise; + + buildWallets(height: number): Promise; + + saveWallets(force: boolean): Promise; + + saveBlock(block: models.Block): Promise; + + // TODO: These methods are exposing database terminology on the business layer, not a fan... + + enqueueSaveBlock(block: models.Block): void; + + enqueueDeleteBlock(block: models.Block): void; + + enqueueDeleteRound(height: number): void; + + commitQueuedQueries(): Promise; + + deleteBlock(block: models.Block): Promise; + + getBlock(id: string): Promise; + + getLastBlock(): Promise; + + getBlocks(offset: number, limit: number): Promise; + + getTopBlocks(count): Promise; + + getRecentBlockIds(): Promise; + + saveRound(activeDelegates: object[]): Promise; + + deleteRound(round: any): Promise; + + getTransaction(id: string): Promise; + + getForgedTransactionsIds(ids: string[]): Promise; + + init(): Promise; + + loadBlocksFromCurrentRound(): Promise; + + loadTransactionsForBlocks(blocks): Promise; + + updateDelegateStats(delegates: any[]): void; + + applyRound(height: number): Promise; + + revertRound(height: number): Promise; + + applyBlock(block: models.Block): Promise; + + revertBlock(block: models.Block): Promise; + + verifyTransaction(transaction: models.Transaction): Promise; + + getBlocksForRound(round?: number): Promise; + + getCommonBlocks(ids: string[]): Promise; +} diff --git a/packages/core-interfaces/src/core-database/event-types.ts b/packages/core-interfaces/src/core-database/event-types.ts new file mode 100644 index 0000000000..4f16f870a4 --- /dev/null +++ b/packages/core-interfaces/src/core-database/event-types.ts @@ -0,0 +1,6 @@ +export enum DatabaseEvents { + PRE_CONNECT = "database.pre_connect", + POST_CONNECT = "database.post_connect", + PRE_DISCONNECT = "database.pre_disconnect", + POST_DISCONNECT = "databse.post_disconnect" +} diff --git a/packages/core-interfaces/src/core-database/index.ts b/packages/core-interfaces/src/core-database/index.ts new file mode 100644 index 0000000000..fd44d022fe --- /dev/null +++ b/packages/core-interfaces/src/core-database/index.ts @@ -0,0 +1,6 @@ +export * from "./database-repository"; +export * from "./business-repository"; +export * from "./database-connection"; +export * from "./wallet-manager"; +export * from "./database-service"; +export * from "./event-types"; diff --git a/packages/core-interfaces/src/core-database/wallet-manager.ts b/packages/core-interfaces/src/core-database/wallet-manager.ts new file mode 100644 index 0000000000..565089b195 --- /dev/null +++ b/packages/core-interfaces/src/core-database/wallet-manager.ts @@ -0,0 +1,61 @@ +import { models } from "@arkecosystem/crypto"; +import { Logger } from "../index"; + +export interface IWalletManager { + + logger: Logger.ILogger; + + config: any; + + reset(): void; + + allByAddress(): models.Wallet[]; + + allByPublicKey(): models.Wallet[]; + + allByUsername(): models.Wallet[]; + + findByAddress(address: string): models.Wallet; + + exists(addressOrPublicKey: string): boolean; + + findByPublicKey(publicKey: string): models.Wallet; + + findByUsername(username: string): models.Wallet; + + index(wallets: models.Wallet[]): void; + + reindex(wallet: models.Wallet): void; + + clear(): void; + + loadActiveDelegateList(maxDelegateCount: number, height?: number): any[]; + + buildVoteBalances(): void; + + applyBlock(block: models.Block): void; + + revertBlock(block: models.Block): void; + + applyTransaction(transaction: models.Transaction): models.Transaction; + + revertTransaction(transaction: models.Transaction): any; + + isDelegate(publicKey: string): boolean; + + canBePurged(wallet: models.Wallet): boolean; + + forgetByAddress(address: string): void; + + forgetByPublicKey( publicKey: string): void; + + forgetByUsername(username: string): void; + + setByAddress(address: string, wallet: models.Wallet): void; + + setByPublicKey(publicKey: string, wallet: models.Wallet): void; + + setByUsername(username: string, wallet: models.Wallet): void; + + purgeEmptyNonDelegates(): void; +} diff --git a/packages/core-interfaces/src/index.ts b/packages/core-interfaces/src/index.ts index 13d8c6faad..367e0a230a 100644 --- a/packages/core-interfaces/src/index.ts +++ b/packages/core-interfaces/src/index.ts @@ -1,9 +1,10 @@ import * as Blockchain from "./core-blockchain"; import * as Container from "./core-container"; +import * as Database from "./core-database"; import * as EventEmitter from "./core-event-emitter"; import * as Logger from "./core-logger"; import * as P2P from "./core-p2p"; import * as TransactionPool from "./core-transaction-pool"; import * as Shared from "./shared"; -export { Container, Logger, Blockchain, TransactionPool, Shared, EventEmitter, P2P }; +export { Container, Logger, Blockchain, TransactionPool, Shared, EventEmitter, P2P, Database }; diff --git a/packages/core-p2p/__tests__/__support__/setup.ts b/packages/core-p2p/__tests__/__support__/setup.ts index b43d500e5c..2c4e2bcfd6 100644 --- a/packages/core-p2p/__tests__/__support__/setup.ts +++ b/packages/core-p2p/__tests__/__support__/setup.ts @@ -1,5 +1,4 @@ import { app } from "@arkecosystem/core-container"; -import delay from "delay"; import { registerWithContainer, setUpContainer } from "../../../core-test-utils/src/helpers/container"; jest.setTimeout(60000); @@ -18,29 +17,13 @@ export const setUp = async () => { }); // register p2p plugin - const { plugin } = require("../../src/plugin"); - await registerWithContainer(plugin, options); - - // and now register blockchain as it has to be registered after p2p - // a little trick here, we register blockchain plugin without starting it - // (it caused some issues where we waited eternally for blockchain to be up) - // instead, we start blockchain manually and check manually that it is up with getLastBlock() - process.env.CORE_SKIP_BLOCKCHAIN = "true"; - const { plugin: pluginBlockchain } = require("@arkecosystem/core-blockchain"); - const blockchain = await registerWithContainer(pluginBlockchain, {}); - await blockchain.start(true); - - while (!blockchain.getLastBlock()) { - await delay(1000); - } + await registerWithContainer(require("../../src/plugin").plugin, options); + await registerWithContainer(require("@arkecosystem/core-blockchain").plugin, {}); }; export const tearDown = async () => { - const { plugin: pluginBlockchain } = require("@arkecosystem/core-blockchain"); - await pluginBlockchain.deregister(app, {}); - - const { plugin } = require("../../src/plugin"); - await plugin.deregister(app, options); + await require("@arkecosystem/core-blockchain").plugin.deregister(app, {}); + await require("../../src/plugin").plugin.deregister(app, options); await app.tearDown(); }; diff --git a/packages/core-p2p/package.json b/packages/core-p2p/package.json index 95d9d1bb39..50e588ac07 100644 --- a/packages/core-p2p/package.json +++ b/packages/core-p2p/package.json @@ -33,7 +33,6 @@ "dependencies": { "@arkecosystem/core-interfaces": "^2.1.0", "@arkecosystem/core-container": "^2.1.0", - "@arkecosystem/core-database-postgres": "^2.1.0", "@arkecosystem/core-http-utils": "^2.1.0", "@arkecosystem/core-transaction-pool": "^2.1.0", "@arkecosystem/crypto": "^2.1.0", diff --git a/packages/core-p2p/src/monitor.ts b/packages/core-p2p/src/monitor.ts index 7d9707c0b1..3697e7d4ab 100644 --- a/packages/core-p2p/src/monitor.ts +++ b/packages/core-p2p/src/monitor.ts @@ -1,8 +1,7 @@ /* tslint:disable:max-line-length */ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain, EventEmitter, Logger, P2P } from "@arkecosystem/core-interfaces"; +import { Blockchain, Database, EventEmitter, Logger, P2P } from "@arkecosystem/core-interfaces"; import { slots } from "@arkecosystem/crypto"; import dayjs from "dayjs-ext"; import delay from "delay"; @@ -716,7 +715,7 @@ export class Monitor implements P2P.IMonitor { * @return {[]String} */ public async __getRecentBlockIds() { - return app.resolvePlugin("database").getRecentBlockIds(); + return app.resolvePlugin("database").getRecentBlockIds(); } /** diff --git a/packages/core-p2p/src/server/versions/1/handlers.ts b/packages/core-p2p/src/server/versions/1/handlers.ts index ef76c9eb30..6f3a1fed6a 100644 --- a/packages/core-p2p/src/server/versions/1/handlers.ts +++ b/packages/core-p2p/src/server/versions/1/handlers.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain, Logger, P2P } from "@arkecosystem/core-interfaces"; +import { Blockchain, Database, Logger, P2P } from "@arkecosystem/core-interfaces"; import { TransactionGuard, TransactionPool } from "@arkecosystem/core-transaction-pool"; import { Joi, models, slots } from "@arkecosystem/crypto"; @@ -250,7 +249,7 @@ export const getBlocks = { */ async handler(request, h) { try { - const database = app.resolvePlugin("database"); + const databaseService = app.resolvePlugin("database"); const blockchain = app.resolvePlugin("blockchain"); const reqBlockHeight = +request.query.lastBlockHeight + 1; @@ -259,7 +258,7 @@ export const getBlocks = { if (!request.query.lastBlockHeight || isNaN(reqBlockHeight)) { blocks.push(blockchain.getLastBlock()); } else { - blocks = await database.getBlocks(reqBlockHeight, 400); + blocks = await databaseService.getBlocks(reqBlockHeight, 400); } logger.info( diff --git a/packages/core-p2p/src/server/versions/internal/handlers/rounds.ts b/packages/core-p2p/src/server/versions/internal/handlers/rounds.ts index bd9a2b6e6c..61ba6a1731 100644 --- a/packages/core-p2p/src/server/versions/internal/handlers/rounds.ts +++ b/packages/core-p2p/src/server/versions/internal/handlers/rounds.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain } from "@arkecosystem/core-interfaces"; +import { Blockchain, Database } from "@arkecosystem/core-interfaces"; import { slots } from "@arkecosystem/crypto"; const config = app.getConfig(); @@ -15,7 +14,7 @@ export const current = { * @return {Hapi.Response} */ async handler(request, h) { - const database = app.resolvePlugin("database"); + const databaseService = app.resolvePlugin("database"); const blockchain = app.resolvePlugin("blockchain"); const lastBlock = blockchain.getLastBlock(); @@ -24,7 +23,7 @@ export const current = { const maxActive = config.getMilestone(height).activeDelegates; const blockTime = config.getMilestone(height).blocktime; const reward = config.getMilestone(height).reward; - const delegates = await database.getActiveDelegates(height); + const delegates = await databaseService.getActiveDelegates(height); const timestamp = slots.getTime(); const currentForger = parseInt((timestamp / blockTime) as any) % maxActive; diff --git a/packages/core-p2p/src/server/versions/internal/handlers/transactions.ts b/packages/core-p2p/src/server/versions/internal/handlers/transactions.ts index 1c299cdfe7..8fb6e0c9ff 100644 --- a/packages/core-p2p/src/server/versions/internal/handlers/transactions.ts +++ b/packages/core-p2p/src/server/versions/internal/handlers/transactions.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain } from "@arkecosystem/core-interfaces"; +import { Blockchain, Database } from "@arkecosystem/core-interfaces"; import { models } from "@arkecosystem/crypto"; import * as schema from "../schemas/transactions"; @@ -21,7 +20,7 @@ export const verify = { return { data: { - valid: await app.resolvePlugin("database").verifyTransaction(transaction), + valid: await app.resolvePlugin("database").verifyTransaction(transaction), }, }; }, diff --git a/packages/core-snapshots/src/db/index.ts b/packages/core-snapshots/src/db/index.ts index 81c23125d7..029cf0223a 100644 --- a/packages/core-snapshots/src/db/index.ts +++ b/packages/core-snapshots/src/db/index.ts @@ -1,5 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { migrations, plugin } from "@arkecosystem/core-database-postgres"; +import { migrations, plugin, PostgresConnection } from "@arkecosystem/core-database-postgres"; import { Logger } from "@arkecosystem/core-interfaces"; import promise from "bluebird"; @@ -16,10 +16,10 @@ class Database { public blocksColumnSet: any; public transactionsColumnSet: any; - public async make(connection) { + public async make(connection : PostgresConnection) { if (connection) { this.db = connection.db; - this.pgp = connection.pgp; + this.pgp = (connection as any).pgp; this.__createColumnSets(); this.isSharedConnection = true; logger.info("Snapshots: reusing core-database-postgres connection from running core"); diff --git a/packages/core-snapshots/src/manager.ts b/packages/core-snapshots/src/manager.ts index 05e3bf1345..0e872d2305 100644 --- a/packages/core-snapshots/src/manager.ts +++ b/packages/core-snapshots/src/manager.ts @@ -1,6 +1,7 @@ /* tslint:disable:max-line-length */ import { app } from "@arkecosystem/core-container"; +import { PostgresConnection } from "@arkecosystem/core-database-postgres"; import { Logger } from "@arkecosystem/core-interfaces"; import pick from "lodash/pick"; @@ -14,7 +15,7 @@ export class SnapshotManager { public database: any; constructor(readonly options) {} - public async make(connection) { + public async make(connection: PostgresConnection) { this.database = await database.make(connection); return this; diff --git a/packages/core-snapshots/src/plugin.ts b/packages/core-snapshots/src/plugin.ts index 88319f6f5f..4ab3891387 100644 --- a/packages/core-snapshots/src/plugin.ts +++ b/packages/core-snapshots/src/plugin.ts @@ -1,5 +1,5 @@ import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Container } from "@arkecosystem/core-interfaces"; +import { Container, Database } from "@arkecosystem/core-interfaces"; import { defaults } from "./defaults"; import { SnapshotManager } from "./manager"; @@ -14,6 +14,11 @@ export const plugin: Container.PluginDescriptor = { async register(container: Container.IContainer, options) { const manager = new SnapshotManager(options); - return manager.make(container.resolvePlugin("database")); + const databaseService = container.resolvePlugin("database"); + if(!!databaseService) { + const connection = databaseService.connection as any; + return await manager.make(connection as PostgresConnection); + } + return await manager.make(null); }, }; diff --git a/packages/core-test-utils/package.json b/packages/core-test-utils/package.json index f39d5934dc..cc159d27b0 100644 --- a/packages/core-test-utils/package.json +++ b/packages/core-test-utils/package.json @@ -8,7 +8,8 @@ "Joshua Noack " ], "license": "MIT", - "main": "dist/index.js", + "main": "dist/index", + "types": "dist/index", "files": [ "dist" ], diff --git a/packages/core-transaction-pool/__tests__/__support__/setup.ts b/packages/core-transaction-pool/__tests__/__support__/setup.ts index f890845cf6..330882c231 100644 --- a/packages/core-transaction-pool/__tests__/__support__/setup.ts +++ b/packages/core-transaction-pool/__tests__/__support__/setup.ts @@ -1,5 +1,4 @@ import { app } from "@arkecosystem/core-container"; -import delay from "delay"; import { registerWithContainer, setUpContainer } from "../../../core-test-utils/src/helpers/container"; jest.setTimeout(60000); @@ -41,32 +40,17 @@ export const setUpFull = async () => { network: "unitnet", }); - const { plugin } = require("../../src/plugin"); - await registerWithContainer(plugin, options); + await registerWithContainer(require("../../src/plugin").plugin, options); // now registering the plugins that need to be registered after transaction pool // register p2p - const { plugin: pluginP2p } = require("@arkecosystem/core-p2p"); - await registerWithContainer(pluginP2p, { + await registerWithContainer(require("@arkecosystem/core-p2p").plugin, { host: "0.0.0.0", port: 4000, minimumNetworkReach: 5, coldStart: 5, }); - - // register blockchain - // a little trick here, we register blockchain plugin without starting it - // (it caused some issues where we waited eternally for blockchain to be up) - // instead, we start blockchain manually and check manually that it is up with getLastBlock() - process.env.CORE_SKIP_BLOCKCHAIN = "true"; - const { plugin: pluginBlockchain } = require("@arkecosystem/core-blockchain"); - const blockchain = await registerWithContainer(pluginBlockchain, {}); - await blockchain.start(true); - - while (!blockchain.getLastBlock()) { - await delay(1000); - } - + await registerWithContainer(require("@arkecosystem/core-blockchain").plugin, {}); return app; }; @@ -75,11 +59,9 @@ export const tearDown = async () => { }; export const tearDownFull = async () => { - const { plugin: pluginP2p } = require("@arkecosystem/core-p2p"); - await pluginP2p.deregister(app, {}); - - const { plugin } = require("../../src/plugin"); - await plugin.deregister(app, options); + await require("../../src/plugin").plugin.deregister(app, options); + await require("@arkecosystem/core-p2p").plugin.deregister(app, {}); + await require("@arkecosystem/core-blockchain").plugin.deregister(app, {}); await app.tearDown(); }; diff --git a/packages/core-transaction-pool/__tests__/connection.test.ts b/packages/core-transaction-pool/__tests__/connection.test.ts index 920c53f1d0..77ebebcbd4 100644 --- a/packages/core-transaction-pool/__tests__/connection.test.ts +++ b/packages/core-transaction-pool/__tests__/connection.test.ts @@ -1,6 +1,6 @@ /* tslint:disable:max-line-length */ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; import { bignumify } from "@arkecosystem/core-utils"; import { Bignum, constants, models, slots } from "@arkecosystem/crypto"; import dayjs from "dayjs-ext"; @@ -18,19 +18,19 @@ const { generateTransfers } = generators; const delegatesSecrets = delegates.map(d => d.secret); let config; -let database: PostgresConnection; +let databaseService: Database.IDatabaseService; let connection: TransactionPool; beforeAll(async () => { await setUpFull(); config = app.getConfig(); - database = app.resolvePlugin("database"); + databaseService = app.resolvePlugin("database"); connection = app.resolvePlugin("transactionPool"); // Ensure no cold wallet and enough funds - database.walletManager.findByPublicKey("000000000000000000000000000000000000000420000000000000000000000000"); - database.walletManager.findByPublicKey( + databaseService.walletManager.findByPublicKey("000000000000000000000000000000000000000420000000000000000000000000"); + databaseService.walletManager.findByPublicKey( "0310c283aac7b35b4ae6fab201d36e8322c3408331149982e16013a5bcb917081c", ).balance = bignumify(200 * 1e8); @@ -206,7 +206,7 @@ describe("Connection", () => { transactions.push(mockData.dummy2); // Ensure no cold wallets - transactions.forEach(tx => database.walletManager.findByPublicKey(tx.senderPublicKey)); + transactions.forEach(tx => databaseService.walletManager.findByPublicKey(tx.senderPublicKey)); const { added, notAdded } = connection.addTransactions(transactions); expect(added).toHaveLength(4); @@ -473,12 +473,12 @@ describe("Connection", () => { const senderRecipientWallet = connection.walletManager.findByAddress(block2.transactions[0].recipientId); senderRecipientWallet.balance = new Bignum(10); // not enough funds for transactions in block - expect(connection.walletManager.all()).toEqual([senderRecipientWallet]); + expect(connection.walletManager.allByAddress()).toEqual([senderRecipientWallet]); // canApply should fail because wallet has not enough funds connection.acceptChainedBlock(new Block(block2)); - expect(connection.walletManager.all()).toEqual([]); + expect(connection.walletManager.allByAddress()).toEqual([]); expect(connection.isSenderBlocked(block2.transactions[0].senderPublicKey)).toBeTrue(); }); @@ -486,11 +486,11 @@ describe("Connection", () => { const senderRecipientWallet = connection.walletManager.findByAddress(block2.transactions[0].recipientId); senderRecipientWallet.balance = new Bignum(block2.totalFee); // exactly enough funds for transactions in block - expect(connection.walletManager.all()).toEqual([senderRecipientWallet]); + expect(connection.walletManager.allByAddress()).toEqual([senderRecipientWallet]); connection.acceptChainedBlock(new Block(block2)); - expect(connection.walletManager.all()).toEqual([]); + expect(connection.walletManager.allByAddress()).toEqual([]); }); }); @@ -506,11 +506,11 @@ describe("Connection", () => { connection.walletManager.reset(); - expect(connection.walletManager.all()).toEqual([]); + expect(connection.walletManager.allByAddress()).toEqual([]); await connection.buildWallets(); - const allWallets = connection.walletManager.all(); + const allWallets = connection.walletManager.allByAddress(); expect(allWallets).toHaveLength(1); expect(allWallets[0].publicKey).toBe(transaction0.senderPublicKey); }); @@ -523,13 +523,13 @@ describe("Connection", () => { connection.walletManager.reset(); - expect(connection.walletManager.all()).toEqual([]); + expect(connection.walletManager.allByAddress()).toEqual([]); jest.spyOn(connection, "getTransaction").mockImplementationOnce(id => undefined); await connection.buildWallets(); - expect(connection.walletManager.all()).toEqual([]); + expect(connection.walletManager.allByAddress()).toEqual([]); }); it("should not apply transaction to wallet if canApply() failed", async () => { @@ -538,7 +538,7 @@ describe("Connection", () => { expect(connection.getTransactions(0, 10)).toEqual([transaction0.serialized]); connection.walletManager.reset(); - expect(connection.walletManager.all()).toEqual([]); + expect(connection.walletManager.allByAddress()).toEqual([]); const senderRecipientWallet = connection.walletManager.findByAddress(block2.transactions[0].recipientId); senderRecipientWallet.balance = new Bignum(10); // not enough funds for transactions in block @@ -549,7 +549,7 @@ describe("Connection", () => { await connection.buildWallets(); - expect(connection.walletManager.all()).toEqual([]); // canApply() failed, wallet was purged + expect(connection.walletManager.allByAddress()).toEqual([]); // canApply() failed, wallet was purged }); }); @@ -614,7 +614,7 @@ describe("Connection", () => { it("remove forged when starting", async () => { expect(connection.getPoolSize()).toBe(0); - const block = await database.getLastBlock(); + const block = await databaseService.getLastBlock(); // XXX This accesses directly block.transactions which is not even // documented in packages/crypto/src/models/block.js @@ -626,8 +626,8 @@ describe("Connection", () => { // For some reason all genesis transactions fail signature verification, so // they are not loaded from the local storage and this fails otherwise. // TODO: Use jest.spyOn() to change behavior instead. jest.restoreAllMocks() will reset afterwards - const original = database.getForgedTransactionsIds; - database.getForgedTransactionsIds = jest.fn(() => [forgedTransaction.id]); + const original = databaseService.getForgedTransactionsIds; + databaseService.getForgedTransactionsIds = jest.fn(() => [forgedTransaction.id]); expect(forgedTransaction instanceof Transaction).toBeTrue(); @@ -651,7 +651,7 @@ describe("Connection", () => { connection.flush(); - database.getForgedTransactionsIds = original; + databaseService.getForgedTransactionsIds = original; }); }); diff --git a/packages/core-transaction-pool/__tests__/pool-wallet-manager.test.ts b/packages/core-transaction-pool/__tests__/pool-wallet-manager.test.ts index 072dbf9d4b..1c67dd8526 100644 --- a/packages/core-transaction-pool/__tests__/pool-wallet-manager.test.ts +++ b/packages/core-transaction-pool/__tests__/pool-wallet-manager.test.ts @@ -1,5 +1,4 @@ -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain, Container } from "@arkecosystem/core-interfaces"; +import { Blockchain, Container, Database } from "@arkecosystem/core-interfaces"; import { generators } from "@arkecosystem/core-test-utils"; import { delegates, genesisBlock, wallets } from "@arkecosystem/core-test-utils/src/fixtures/unitnet"; import { crypto, models } from "@arkecosystem/crypto"; @@ -123,7 +122,7 @@ describe("applyPoolTransactionToSender", () => { // This is normally refused because it's a cold wallet, but since we want // to test if chained transfers are refused, pretent it is not a cold wallet. container - .resolvePlugin("database") + .resolvePlugin("database") .walletManager.findByPublicKey(transfer.senderPublicKey); const errors = []; @@ -140,9 +139,9 @@ describe("applyPoolTransactionToSender", () => { ); } - container - .resolvePlugin("database") - .walletManager.forgetByPublicKey(transfer.publicKey); + (container.resolvePlugin("database").walletManager as any).forgetByPublicKey( + transfer.publicKey, + ); }); expect(+delegateWallet.balance).toBe(delegate.balance - (100 + 0.1) * arktoshi); diff --git a/packages/core-transaction-pool/package.json b/packages/core-transaction-pool/package.json index a36f19c619..ac05693c10 100644 --- a/packages/core-transaction-pool/package.json +++ b/packages/core-transaction-pool/package.json @@ -34,7 +34,6 @@ "dependencies": { "@arkecosystem/core-container": "^2.1.0", "@arkecosystem/core-database": "^2.1.0", - "@arkecosystem/core-database-postgres": "^2.1.0", "@arkecosystem/crypto": "^2.1.0", "@arkecosystem/core-interfaces": "^2.1.0", "@types/better-sqlite3": "^5.2.0", diff --git a/packages/core-transaction-pool/src/connection.ts b/packages/core-transaction-pool/src/connection.ts index 79039a5c9a..8713b69b52 100644 --- a/packages/core-transaction-pool/src/connection.ts +++ b/packages/core-transaction-pool/src/connection.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { EventEmitter, Logger, TransactionPool as transactionPool } from "@arkecosystem/core-interfaces"; +import { Database, EventEmitter, Logger, TransactionPool as transactionPool } from "@arkecosystem/core-interfaces"; import assert from "assert"; import dayjs from "dayjs-ext"; @@ -10,7 +9,7 @@ import { Mem } from "./mem"; import { MemPoolTransaction } from "./mem-pool-transaction"; import { Storage } from "./storage"; -const database = app.resolvePlugin("database"); +const databaseService = app.resolvePlugin("database"); const emitter = app.resolvePlugin("event-emitter"); const logger = app.resolvePlugin("logger"); @@ -54,7 +53,7 @@ export class TransactionPool implements transactionPool.ITransactionPool { // Remove transactions that were forged while we were offline. const allIds = all.map(memPoolTransaction => memPoolTransaction.transaction.id); - const forgedIds = await database.getForgedTransactionsIds(allIds); + const forgedIds = await databaseService.getForgedTransactionsIds(allIds); forgedIds.forEach(id => this.removeTransactionById(id)); @@ -416,7 +415,7 @@ export class TransactionPool implements transactionPool.ITransactionPool { if ( senderWallet && - this.walletManager.__canBePurged(senderWallet) && + this.walletManager.canBePurged(senderWallet) && this.getSenderSize(senderPublicKey) === 0 ) { this.walletManager.deleteWallet(senderPublicKey); diff --git a/packages/core-transaction-pool/src/guard.ts b/packages/core-transaction-pool/src/guard.ts index c715b1bc48..9c6cb76c62 100644 --- a/packages/core-transaction-pool/src/guard.ts +++ b/packages/core-transaction-pool/src/guard.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Logger, TransactionPool as transanctionPool } from "@arkecosystem/core-interfaces"; +import { Database, Logger, TransactionPool as transactionPool } from "@arkecosystem/core-interfaces"; import { configManager, constants, models, slots } from "@arkecosystem/crypto"; import pluralize from "pluralize"; import { TransactionPool } from "./connection"; @@ -10,13 +9,13 @@ import { isRecipientOnActiveNetwork } from "./utils/is-on-active-network"; const { TransactionTypes } = constants; const { Transaction } = models; -export class TransactionGuard implements transanctionPool.ITransactionGuard { +export class TransactionGuard implements transactionPool.ITransactionGuard { public transactions: models.Transaction[] = []; public excess: string[] = []; public accept: Map = new Map(); public broadcast: Map = new Map(); public invalid: Map = new Map(); - public errors: { [key: string]: transanctionPool.TransactionErrorDTO[] } = {}; + public errors: { [key: string]: transactionPool.TransactionErrorDTO[] } = {}; /** * Create a new transaction guard instance. @@ -38,7 +37,7 @@ export class TransactionGuard implements transanctionPool.ITransactionGuard { * value=[ { type, message }, ... ] * } */ - public async validate(transactions: models.Transaction[]): Promise { + public async validate(transactions: models.Transaction[]): Promise { this.pool.loggedAllowedSenders = []; // Cache transactions @@ -237,9 +236,9 @@ export class TransactionGuard implements transanctionPool.ITransactionGuard { * @return {void} */ public async __removeForgedTransactions() { - const database = app.resolvePlugin("database"); + const databaseService = app.resolvePlugin("database"); - const forgedIdsSet = await database.getForgedTransactionsIds([ + const forgedIdsSet = await databaseService.getForgedTransactionsIds([ ...new Set([...this.accept.keys(), ...this.broadcast.keys()]), ]); diff --git a/packages/core-transaction-pool/src/pool-wallet-manager.ts b/packages/core-transaction-pool/src/pool-wallet-manager.ts index 4381cd5709..167ca5a147 100644 --- a/packages/core-transaction-pool/src/pool-wallet-manager.ts +++ b/packages/core-transaction-pool/src/pool-wallet-manager.ts @@ -1,13 +1,13 @@ import { app } from "@arkecosystem/core-container"; import { WalletManager } from "@arkecosystem/core-database"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; +import { Database } from "@arkecosystem/core-interfaces"; import { constants, crypto, isException, models } from "@arkecosystem/crypto"; const { Wallet } = models; const { TransactionTypes } = constants; export class PoolWalletManager extends WalletManager { - public database = app.resolvePlugin("database"); + public databaseService = app.resolvePlugin("database"); /** * Create a new pool wallet manager instance. @@ -27,7 +27,7 @@ export class PoolWalletManager extends WalletManager { */ public findByAddress(address) { if (!this.byAddress[address]) { - const blockchainWallet = this.database.walletManager.findByAddress(address); + const blockchainWallet = this.databaseService.walletManager.findByAddress(address); const wallet = Object.assign(new Wallet(address), blockchainWallet); // do not modify this.reindex(wallet); @@ -50,10 +50,10 @@ export class PoolWalletManager extends WalletManager { public canApply(transaction, errors) { // Edge case if sender is unknown and has no balance. // NOTE: Check is performed against the database wallet manager. - if (!this.database.walletManager.byPublicKey[transaction.senderPublicKey]) { + if (!this.databaseService.walletManager.exists(transaction.senderPublicKey)) { const senderAddress = crypto.getAddress(transaction.senderPublicKey, this.networkId); - if (this.database.walletManager.findByAddress(senderAddress).balance.isZero()) { + if (this.databaseService.walletManager.findByAddress(senderAddress).balance.isZero()) { errors.push("Cold wallet is not allowed to send until receiving transaction is confirmed."); return false; } @@ -64,7 +64,7 @@ export class PoolWalletManager extends WalletManager { if ( type === TransactionTypes.DelegateRegistration && - this.database.walletManager.byUsername[asset.delegate.username.toLowerCase()] + this.databaseService.walletManager.findByUsername(asset.delegate.username.toLowerCase()) ) { this.logger.error( `[PoolWalletManager] Can't apply transaction ${ @@ -76,7 +76,7 @@ export class PoolWalletManager extends WalletManager { // NOTE: We use the vote public key, because vote transactions have the same sender and recipient. } else if ( type === TransactionTypes.Vote && - !this.database.walletManager.__isDelegate(asset.votes[0].slice(1)) + !this.databaseService.walletManager.isDelegate(asset.votes[0].slice(1)) ) { this.logger.error( `[PoolWalletManager] Can't apply vote transaction: delegate ${ diff --git a/packages/core-vote-report/package.json b/packages/core-vote-report/package.json index 61363e5cba..61de3c13e7 100644 --- a/packages/core-vote-report/package.json +++ b/packages/core-vote-report/package.json @@ -28,7 +28,6 @@ }, "dependencies": { "@arkecosystem/core-container": "^2.1.0", - "@arkecosystem/core-database-postgres": "^2.1.0", "@arkecosystem/core-http-utils": "^2.1.0", "@arkecosystem/core-utils": "^2.1.0", "@arkecosystem/crypto": "^2.1.0", diff --git a/packages/core-vote-report/src/handler.ts b/packages/core-vote-report/src/handler.ts index 0ef0bc5582..79d91da923 100644 --- a/packages/core-vote-report/src/handler.ts +++ b/packages/core-vote-report/src/handler.ts @@ -1,6 +1,5 @@ import { app } from "@arkecosystem/core-container"; -import { PostgresConnection } from "@arkecosystem/core-database-postgres"; -import { Blockchain } from "@arkecosystem/core-interfaces"; +import { Blockchain, Database } from "@arkecosystem/core-interfaces"; import { delegateCalculator, supplyCalculator } from "@arkecosystem/core-utils"; import { configManager } from "@arkecosystem/crypto"; import sumBy from "lodash/sumBy"; @@ -8,13 +7,13 @@ import sumBy from "lodash/sumBy"; export function handler(request, h) { const config = app.getConfig(); const blockchain = app.resolvePlugin("blockchain"); - const database = app.resolvePlugin("database"); + const databaseService = app.resolvePlugin("database"); const formatDelegates = (delegates, lastHeight) => delegates.map((delegate, index) => { - const filteredVoters = database.walletManager + const filteredVoters = databaseService.walletManager .allByPublicKey() - .filter(wallet => wallet.vote === delegate.publicKey && wallet.balance > 0.1 * 1e8); + .filter(wallet => wallet.vote === delegate.publicKey && wallet.balance.toNumber() > 0.1 * 1e8); const approval = Number(delegateCalculator.calculateApproval(delegate, lastHeight)).toLocaleString( undefined, @@ -49,18 +48,18 @@ export function handler(request, h) { const supply = supplyCalculator.calculate(lastBlock.data.height); - const allByUsername = database.walletManager + const allByUsername = databaseService.walletManager .allByUsername() .map((delegate, index) => { - delegate.rate = delegate.rate || index + 1; + (delegate as any).rate = (delegate as any).rate || index + 1; return delegate; }) - .sort((a, b) => a.rate - b.rate); + .sort((a, b) => (a as any).rate - (b as any).rate); const active = allByUsername.slice(0, constants.activeDelegates); const standby = allByUsername.slice(constants.activeDelegates + 1, delegateRows); - const voters = database.walletManager.allByPublicKey().filter(wallet => wallet.vote && wallet.balance > 0.1 * 1e8); + const voters = databaseService.walletManager.allByPublicKey().filter(wallet => wallet.vote && wallet.balance.toNumber() > 0.1 * 1e8); const totalVotes = sumBy(voters, (wallet: any) => +wallet.balance.toFixed()); const percentage = (totalVotes * 100) / supply;