Skip to content

Commit

Permalink
added watchguard on fillPool and some new tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
vekexasia committed Jun 1, 2018
1 parent 8e2a04f commit e72277c
Show file tree
Hide file tree
Showing 7 changed files with 171 additions and 23 deletions.
7 changes: 5 additions & 2 deletions src/modules/blocks/chain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export class BlocksModuleChain implements IBlocksModuleChain {
* Lock for processing.
* @type {boolean}
*/
private isCleaning: boolean = false;
private isCleaning: boolean = false;

private isProcessing: boolean = false;

Expand Down Expand Up @@ -173,7 +173,10 @@ export class BlocksModuleChain implements IBlocksModuleChain {
}

@WrapInBalanceSequence
public async applyBlock(block: SignedAndChainedBlockType, broadcast: boolean, saveBlock: boolean, accountsMap: { [address: string]: AccountsModel }) {
public async applyBlock(block: SignedAndChainedBlockType,
broadcast: boolean,
saveBlock: boolean,
accountsMap: { [address: string]: AccountsModel }) {
if (this.isCleaning) {
return; // Avoid processing a new block if it is cleaning.
}
Expand Down
8 changes: 8 additions & 0 deletions src/modules/transactions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,14 @@ export class TransactionsModule implements ITransactionsModule {
*/
public async fillPool(): Promise<void> {
const newUnconfirmedTXs = await this.transactionPool.fillPool();
const ids = newUnconfirmedTXs.map((tx) => tx.id);
const alreadyConfirmedIDs = await this.filterConfirmedIds(ids);
for (const confirmedID of alreadyConfirmedIDs) {
this.logger.debug(`TX ${confirmedID} was already confirmed but still in pool`);
this.removeUnconfirmedTransaction(confirmedID);
const idx = newUnconfirmedTXs.findIndex((a) => a.id === confirmedID);
newUnconfirmedTXs.splice(idx, 1);
}
await this.transactionPool.applyUnconfirmedList(newUnconfirmedTXs, this);
}

Expand Down
9 changes: 7 additions & 2 deletions tests/benchmarks/tps.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { Sequelize } from 'sequelize-typescript';
import { IBlocksModule, ISystemModule } from '../../src/ioc/interfaces/modules';
import { reportedIT } from './benchutils';

const numTransactions = 15000;
const numTransactions = 9000;
describe('TPS', function () {
this.timeout(1000000);
initializer.setup();
Expand Down Expand Up @@ -110,7 +110,12 @@ describe('TPS', function () {
// return txs.length / took * 1000;
// });

reportedIT('with always same (non-voting) sender accounts', [25, 50, 100, 200, 300, 400, 500, 1000, 2000, 3000], async (blockSize) => {
reportedIT('with always same (non-voting) sender accounts', [
// 25, 50, 100, 200, 300, 400, 500,
//1000
//, 2000,
3000
], async (blockSize) => {
const txs = [];
const oldTxsPerBlock = consts.maxTxsPerBlock;
consts.maxTxsPerBlock = blockSize;
Expand Down
104 changes: 104 additions & 0 deletions tests/integration/blockProcessing.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import { expect } from 'chai';
import {
createRandomAccountsWithFunds,
createRegDelegateTransaction, createSecondSignTransaction, createVoteTransaction,
easyCreateMultiSignAccount
} from './common/utils';
import { LiskWallet } from 'dpos-offline/dist/es5/liskWallet';
import { ITransaction } from 'dpos-offline/src/trxTypes/BaseTx';
import { BlocksModule } from '../../src/modules';
import initializer from './common/init';
import { Symbols } from '../../src/ioc/symbols';
import { SignedAndChainedBlockType } from '../../src/logic';
import { BlocksModuleChain } from '../../src/modules/blocks/';
import * as supertest from 'supertest';

describe('blockProcessing', async () => {
let blocksModule: BlocksModule;
let blocksChainModule: BlocksModuleChain;
initializer.setup();
beforeEach(() => {
blocksModule = initializer.appManager.container.get(Symbols.modules.blocks);
blocksChainModule = initializer.appManager.container.get(Symbols.modules.blocksSubModules.chain);
});
describe('delete block', () => {
let creationOps: Array<{ tx: ITransaction, account: LiskWallet, senderWallet: LiskWallet }>;
let multisigOp: { wallet: LiskWallet, keys: LiskWallet[], tx: ITransaction};
let block: SignedAndChainedBlockType;
let initBlock: SignedAndChainedBlockType;
let regDelegateTX: ITransaction;
let secondSignTX: ITransaction;
let voteTX: ITransaction;
let allTxs: ITransaction[];
let allAccounts: LiskWallet[];
initializer.autoRestoreEach();
beforeEach(async () => {
initBlock = blocksModule.lastBlock;
creationOps = await createRandomAccountsWithFunds(10, 10e10);
multisigOp = await easyCreateMultiSignAccount(3, 2);
regDelegateTX = await createRegDelegateTransaction(1, creationOps[0].account, 'meow');
secondSignTX = await createSecondSignTransaction(1, creationOps[1].account, creationOps[2].account.publicKey);
voteTX = await createVoteTransaction(1, creationOps[0].account, creationOps[0].account.publicKey, true);

block = blocksModule.lastBlock;
allAccounts = creationOps.map((op) => op.account)
.concat(multisigOp.wallet);
allTxs = creationOps.map((op) => op.tx)
.concat(multisigOp.tx)
.concat(regDelegateTX)
.concat(secondSignTX)
.concat(voteTX);

});
it('should remove block from db', async () => {
for (let i = 0; i < block.height - initBlock.height; i++) {
await blocksChainModule.deleteLastBlock();
}
const b = blocksModule.lastBlock;
expect(blocksModule.lastBlock.height).eq(initBlock.height);
const res = await supertest(initializer.appManager.expressApp)
.get('/api/blocks/get?id=' + block.id)
.expect(200);

expect(res.body.success).is.false;
expect(res.body.error).is.eq('Block not found');
});
it('should remove txs from db', async () => {
expect(block.transactions.length).gt(0);
for (let i = 0; i < block.height - initBlock.height; i++) {
await blocksChainModule.deleteLastBlock();
}

for (const op of allTxs) {
const txID = op.id;

const res = await supertest(initializer.appManager.expressApp)
.get(`/api/transactions/get?id=${txID}`)
.expect(200);
expect(res.body.success).is.false;
expect(res.body.error).is.eq('Transaction not found');
}
});
it('should restore accounts to its original state', async () => {
for (let i = 0; i < block.height - initBlock.height; i++) {
await blocksChainModule.deleteLastBlock();
}

for (const account of allAccounts) {
const res = await supertest(initializer.appManager.expressApp)
.get(`/api/accounts/?address=${account.address}`)
.expect(200);

expect(res.body.success).is.true;
expect(res.body.account.address).is.deep.eq(account.address);
expect(res.body.account.balance).is.deep.eq('0');
expect(res.body.account.username).is.undefined;
expect(res.body.account.multisignatures).is.deep.eq([]);
expect(res.body.account.secondSignature).is.eq(0);
expect(res.body.account.secondPublicKey).is.null;
expect(res.body.account.unconfirmedBalance).is.deep.eq('0');
}
});
});

});
18 changes: 8 additions & 10 deletions tests/integration/highlevel.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,6 @@ describe('highlevel checks', function () {
* instead of sending the "next block" tx we send the current block tx
* in the hope that a dual applyUnconfirmed gets done causing inconsistency.
*/
const jobsQueue = initializer.appManager.container.get<JobsQueue>(Symbols.helpers.jobsQueue);

const fieldheader = {
nethash: 'e4c527bd888c257377c18615d021e9cedd2bc2fd6de04b369f22a8780264c2f6',
Expand All @@ -491,23 +490,22 @@ describe('highlevel checks', function () {
};

const startHeight = blocksModule.lastBlock.height;
const fundPerTx = 1;
const fundPerTx = 1;

// Create some 1 satoshi transactions
const txs = await Promise.all(
new Array(10000).fill(null)
.map((what, idx) => createSendTransaction(0, fundPerTx, senderAccount, '1R', {timestamp: idx}))
);

const total = 1000;
const total = 900;
for (let i = 0; i < total; i++) {
const block = await initializer.generateBlock(txs.slice(i, i + 1));
if (i%(total/10 | 0) === 0) {
if (i % (total / 10 | 0) === 0) {
console.log('Done', i);
}



await Promise.all([
// simulate fillPool (forgeModule calling it)
// wait(Math.random() * 100).then(() => jobsQueue.bau['delegatesNextForge']()),
Expand All @@ -518,13 +516,13 @@ describe('highlevel checks', function () {
.send({block: blocksModel.toStringBlockType(block, txModel, blocksModule)})
.expect(200)),
// Send the current (same) transaction
wait(Math.random()*10).then(() => supertest(initializer.appManager.expressApp)
wait(Math.random() * 10).then(() => supertest(initializer.appManager.expressApp)
.post('/peer/transactions')
.set(fieldheader)
.send({transaction: txs.slice(i, i+1)[0]})
.send({transaction: txs.slice(i, i + 1)[0]})
.expect(200))

]);
]);

expect(blocksModule.lastBlock.blockSignature).to.be.deep.eq(block.blockSignature);

Expand All @@ -536,7 +534,7 @@ describe('highlevel checks', function () {
// Check balances are correct so that no other applyUnconfirmed happened.
// NOTE: this could fail as <<<--HERE-->>> an applyUnconfirmed (of NEXT tx) could
// have happened
const { u_balance, balance } = await accModule.getAccount({address: senderAccount.address});
const {u_balance, balance} = await accModule.getAccount({address: senderAccount.address});
// console.log('End loop - test begins');
expect(new BigNumber(balance).toNumber()).to.be.eq(new BigNumber(funds)
.minus(fundPerTx * (i + 1))
Expand All @@ -545,7 +543,7 @@ describe('highlevel checks', function () {

expect(new BigNumber(u_balance).toNumber()).to.be.eq(new BigNumber(balance).toNumber(), 'unconfirmed balance');

expect(blocksModule.lastBlock.height).to.be.eq( startHeight + i + 1);
expect(blocksModule.lastBlock.height).to.be.eq(startHeight + i + 1);
// console.log('Current Balance is : ', balance, u_balance);
}

Expand Down
21 changes: 15 additions & 6 deletions tests/unit/modules/blocks/chain.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { Symbols } from '../../../../src/ioc/symbols';
import { IBaseTransaction } from '../../../../src/logic/transactions';
import { BlocksModuleChain } from '../../../../src/modules/blocks/';
import { createRandomWallet } from '../../../integration/common/utils';
import { BlocksSubmoduleUtilsStub, BusStub, TransactionsModuleStub } from '../../../stubs';
import { BlocksSubmoduleUtilsStub, BusStub, SequenceStub, TransactionsModuleStub } from '../../../stubs';
import { BlockLogicStub } from '../../../stubs/logic/BlockLogicStub';
import TransactionLogicStub from '../../../stubs/logic/TransactionLogicStub';
import AccountsModuleStub from '../../../stubs/modules/AccountsModuleStub';
Expand Down Expand Up @@ -61,6 +61,8 @@ describe('modules/blocks/chain', () => {
let dbStub: DbStub;
let blocksModel: typeof BlocksModel;
let destroyStub: SinonStub;
let balancesSequence: SequenceStub;

beforeEach(() => {
sandbox = sinon.createSandbox();
accountsModule = container.get(Symbols.modules.accounts);
Expand All @@ -73,7 +75,8 @@ describe('modules/blocks/chain', () => {
blocksModel = container.get(Symbols.models.blocks);
dbStub = container.get(Symbols.helpers.db);
destroyStub = sandbox.stub(blocksModel, 'destroy').resolves();

balancesSequence = container.getTagged(Symbols.helpers.sequence,
Symbols.helpers.sequence, Symbols.tags.helpers.balancesSequence);
busStub = container.get(Symbols.helpers.bus);
});
afterEach(() => sandbox.restore());
Expand Down Expand Up @@ -314,6 +317,11 @@ describe('modules/blocks/chain', () => {
busStub.enqueueResponse('message', Promise.resolve());
txStub = sandbox.stub(blocksModel.sequelize, 'transaction').callsFake((t) => t('tx'));
});
it('should be wrapped in balanceSequence', async () => {
expect(balancesSequence.spies.addAndPromise.called).is.false;
await inst.applyBlock({transactions: allTxs} as any, false, false, accountsMap);
expect(balancesSequence.spies.addAndPromise.called).is.true;
})

it('should skip applyUnconfirmed if txModule.transactionUnconfirmed returns true');
it('should return undefined if cleanup in processing and set instance.isCleaning in true', async () => {
Expand All @@ -322,10 +330,11 @@ describe('modules/blocks/chain', () => {
expect(txModule.stubs.undoUnconfirmedList.notCalled).to.be.true;
});
it('should set .isProcessing to true to prevent shutdowns', async () => {
const p = inst.applyBlock({ transactions: allTxs } as any, false, false, accountsMap);
// tslint:disable-next-line: no-string-literal
expect(inst['isProcessing']).to.be.true;
await p;
txStub.callsFake((t) => {
expect(inst['isProcessing']).to.be.true;
return t('tx');
});
await inst.applyBlock({ transactions: allTxs } as any, false, false, accountsMap);
// tslint:disable-next-line: no-string-literal
expect(inst['isProcessing']).to.be.false;
});
Expand Down
27 changes: 24 additions & 3 deletions tests/unit/modules/transactions.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
import { createContainer } from '../../utils/containerCreator';
import DbStub from '../../stubs/helpers/DbStub';
import { TransactionsModel } from '../../../src/models';
import { createRandomTransactions, toBufferedTransaction } from '../../utils/txCrafter';

chai.use(chaiAsPromised);

Expand Down Expand Up @@ -346,10 +347,13 @@ describe('modules/transactions', () => {
});

describe('fillPool', () => {
const newUnconfirmedTXs = ['tx1', 'tx2'];
const newUnconfirmedTXs = createRandomTransactions({send: 2, vote: 1})
.map((t) => toBufferedTransaction(t));
let filterConfIDsStub: SinonStub;
beforeEach(() => {
transactionPoolStub.stubs.fillPool.resolves(newUnconfirmedTXs);
transactionPoolStub.stubs.fillPool.resolves(newUnconfirmedTXs.slice());
transactionPoolStub.stubs.applyUnconfirmedList.resolves();
filterConfIDsStub = sandbox.stub(instance, 'filterConfirmedIds').resolves([]);
});

it('should call txPool.fillPool', async () => {
Expand All @@ -365,8 +369,25 @@ describe('modules/transactions', () => {
expect(transactionPoolStub.stubs.applyUnconfirmedList.firstCall.args[0]).to.be.deep.equal(newUnconfirmedTXs);
expect(transactionPoolStub.stubs.applyUnconfirmedList.firstCall.args[1]).to.be.deep.equal(instance);
});
});

it('should query for confirmed ids', async () => {
await instance.fillPool();
expect(filterConfIDsStub.called).is.true;
expect(filterConfIDsStub.firstCall.args[0]).is.deep.eq(newUnconfirmedTXs.map((t) => t.id));
});

it('should exclude already confirmed transaction', async () => {
filterConfIDsStub.resolves([newUnconfirmedTXs[1].id]);
await instance.fillPool();
expect(transactionPoolStub.stubs.applyUnconfirmedList.calledOnce).to.be.true;
expect(transactionPoolStub.stubs.applyUnconfirmedList.firstCall.args.length).to.be.equal(2);
expect(transactionPoolStub.stubs.applyUnconfirmedList.firstCall.args[0]).to.be.deep.equal([
newUnconfirmedTXs[0],
newUnconfirmedTXs[2],
]);
expect(transactionPoolStub.stubs.applyUnconfirmedList.firstCall.args[1]).to.be.deep.equal(instance);
});
});

describe('getByID', () => {
let txModel: typeof TransactionsModel;
Expand Down

0 comments on commit e72277c

Please sign in to comment.