Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enabling JournalPruneDataSource #478

Merged
merged 10 commits into from Jun 1, 2018
60 changes: 40 additions & 20 deletions modAionImpl/src/org/aion/zero/impl/AionHub.java
@@ -1,4 +1,4 @@
/*******************************************************************************
/* ******************************************************************************
* Copyright (c) 2017-2018 Aion foundation.
*
* This file is part of the aion network project.
Expand All @@ -19,9 +19,7 @@
*
* Contributors:
* Aion foundation.
*
******************************************************************************/

package org.aion.zero.impl;

import org.aion.base.db.IRepository;
Expand Down Expand Up @@ -272,9 +270,23 @@ private void loadBlockchain() {
bestBlock != null && // recover only for non-null blocks
!this.repository.isValidRoot(bestBlock.getStateRoot())) {

LOG.info("Recovery initiated due to corrupt world state at block " + bestBlock.getNumber() + ".");

long bestBlockNumber = bestBlock.getNumber();
byte[] bestBlockRoot = bestBlock.getStateRoot();

// ensure that the genesis state exists before attempting recovery
AionGenesis genesis = cfg.getGenesis();
if (!this.repository.isValidRoot(genesis.getStateRoot())) {
LOG.info(
"Corrupt world state for genesis block hash: " + genesis.getShortHash() + ", number: " + genesis
.getNumber() + ".");

buildGenesis(genesis);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

buildGenesis() seems to be building the genesis state based on the current repository. It's correct for the latter usage, when the database is empty, but is it still valid here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's valid, i.e. it always results in the same state root, because when calling createAccount(Address) the stateRoot = EMPTY_TRIE_HASH;, so it does not build on the state root of the tracked repository


LOG.info("Rebuilding genesis block SUCCEEDED.");
}

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about extract this code snippet to a mathod?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if buildGenesis fails? Do we need any log message here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if there is anything wrong with the genesis file an error message will be printed before recovery is attempted

recovered = this.blockchain.recoverWorldState(this.repository, bestBlock);

if (!this.repository.isValidRoot(bestBlock.getStateRoot())) {
Expand Down Expand Up @@ -323,24 +335,8 @@ private void loadBlockchain() {

AionGenesis genesis = cfg.getGenesis();

// initialization section for network balance contract
IRepositoryCache track = repository.startTracking();

Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress;
track.createAccount(networkBalanceAddress);

for (Map.Entry<Integer, BigInteger> addr : genesis.getNetworkBalances().entrySet()) {
track.addStorageRow(networkBalanceAddress, new DataWord(addr.getKey()), new DataWord(addr.getValue()));
}
buildGenesis(genesis);

for (Address addr : genesis.getPremine().keySet()) {
track.createAccount(addr);
track.addBalance(addr, genesis.getPremine().get(addr).getBalance());
}
track.flush();

repository.commitBlock(genesis.getHeader());
this.repository.getBlockStore().saveBlock(genesis, genesis.getCumulativeDifficulty(), true);
blockchain.setBestBlock(genesis);
blockchain.setTotalDifficulty(genesis.getCumulativeDifficulty());

Expand Down Expand Up @@ -395,6 +391,30 @@ private void loadBlockchain() {
// this.repository.getBlockStore().load();
}

private void buildGenesis(AionGenesis genesis) {
// initialization section for network balance contract
IRepositoryCache track = repository.startTracking();

Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress;
track.createAccount(networkBalanceAddress);

for (Map.Entry<Integer, BigInteger> addr : genesis.getNetworkBalances().entrySet()) {
track.addStorageRow(
networkBalanceAddress,
new DataWord(addr.getKey()),
new DataWord(addr.getValue()));
}

for (Address addr : genesis.getPremine().keySet()) {
track.createAccount(addr);
track.addBalance(addr, genesis.getPremine().get(addr).getBalance());
}
track.flush();

this.repository.commitBlock(genesis.getHeader());
this.repository.getBlockStore().saveBlock(genesis, genesis.getCumulativeDifficulty(), true);
}

public void close() {
LOG.info("<KERNEL SHUTDOWN SEQUENCE>");

Expand Down
49 changes: 47 additions & 2 deletions modAionImpl/src/org/aion/zero/impl/cli/Cli.java
Expand Up @@ -129,11 +129,52 @@ public int call(final String[] args, final Cfg cfg) {
}
}
break;
case "--dump-state-size":
long block_count = 2L;

if (args.length < 2) {
System.out.println("Retrieving state size for top " + block_count + " blocks.");
RecoveryUtils.printStateTrieSize(block_count);
} else {
try {
block_count = Long.parseLong(args[1]);
} catch (NumberFormatException e) {
System.out.println("The given argument <" + args[1] + "> cannot be converted to a number.");
}
if (block_count < 1) {
System.out.println("The given argument <" + args[1] + "> is not valid.");
block_count = 2L;
}

System.out.println("Retrieving state size for top " + block_count + " blocks.");
RecoveryUtils.printStateTrieSize(block_count);
}
break;
case "--dump-state":
long level = -1L;

if (args.length < 2) {
System.out.println("Retrieving state for top main chain block...");
RecoveryUtils.printStateTrieDump(level);
} else {
try {
level = Long.parseLong(args[1]);
} catch (NumberFormatException e) {
System.out.println("The given argument <" + args[1] + "> cannot be converted to a number.");
}
if (level == -1L) {
System.out.println("Retrieving state for top main chain block...");
} else {
System.out.println("Retrieving state for main chain block at level " + level + "...");
}
RecoveryUtils.printStateTrieDump(level);
}
break;
case "--db-compact":
RecoveryUtils.dbCompact();
break;
case "--dump-blocks":
long count = 100L;
long count = 10L;

if (args.length < 2) {
System.out.println("Printing top " + count + " blocks from database.");
Expand All @@ -144,10 +185,14 @@ public int call(final String[] args, final Cfg cfg) {
} catch (NumberFormatException e) {
System.out.println("The given argument <" + args[1] + "> cannot be converted to a number.");
}
if (count < 1) {
System.out.println("The given argument <" + args[1] + "> is not valid.");
count = 10L;
}

System.out.println("Printing top " + count + " blocks from database.");
RecoveryUtils.dumpBlocks(count);
}
System.out.println("Finished printing blocks.");
break;
case "-v":
System.out.println("\nVersion");
Expand Down
6 changes: 5 additions & 1 deletion modAionImpl/src/org/aion/zero/impl/db/AionBlockStore.java
Expand Up @@ -749,11 +749,14 @@ public BigInteger correctIndexEntry(AionBlock block, BigInteger parentTotalDiffi
}
}

public void dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOException {
public String dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOException {
lock.readLock().lock();

try {
long firstBlock = getMaxNumber();
if (firstBlock < 0) {
return null;
}
long lastBlock = firstBlock - numberOfBlocks;

File file = new File(reportsFolder, System.currentTimeMillis() + "-blocks-report.out");
Expand Down Expand Up @@ -786,6 +789,7 @@ public void dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOE
}

writer.close();
return file.getName();
} finally {
lock.readLock().unlock();
}
Expand Down
36 changes: 19 additions & 17 deletions modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java
Expand Up @@ -76,7 +76,12 @@ private static class AionRepositoryImplHolder {
// repository singleton instance
private final static AionRepositoryImpl inst = new AionRepositoryImpl(
new RepositoryConfig(new File(config.getBasePath(), config.getDb().getPath()).getAbsolutePath(),
-1,
config.getDb().getPrune() > 0 ?
// if the value is smaller than backward step
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pruning value should dictate parameters for our sync. So if we set a value of 16 as our pruning parameter, sync should respect that value.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the interaction between pruning and sync is tuned in #498

// there is the risk of importing state-less blocks after reboot
(128 > config.getDb().getPrune() ? 128 : config.getDb().getPrune()) :
// negative value => pruning disabled
config.getDb().getPrune(),
ContractDetailsAion.getInstance(),
config.getDb()));
}
Expand Down Expand Up @@ -115,7 +120,7 @@ public TransactionStore<AionTransaction, AionTxReceipt, AionTxInfo> getTransacti
}

private Trie createStateTrie() {
return new SecureTrie(stateDatabase).withPruningEnabled(pruneBlockCount >= 0);
return new SecureTrie(stateDSPrune).withPruningEnabled(pruneBlockCount > 0);
}

@Override
Expand Down Expand Up @@ -501,34 +506,31 @@ public void commitBlock(A0BlockHeader blockHeader) {
worldState.sync();
detailsDS.syncLargeStorage();

// temporarily removed since never used
/* if (pruneBlockCount >= 0) {
stateDSPrune.storeBlockChanges(blockHeader);
detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader);
pruneBlocks(blockHeader);
} */
if (pruneBlockCount > 0) {
stateDSPrune.storeBlockChanges(blockHeader);
detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader);
pruneBlocks(blockHeader);
}
} finally {
rwLock.writeLock().unlock();
}
}

// TODO-AR: reenable state pruning
// temporarily removed since never used
/* private void pruneBlocks(A0BlockHeader curBlock) {
if (curBlock.getNumber() > bestBlockNumber) { // pruning only on
// increasing blocks
private void pruneBlocks(A0BlockHeader curBlock) {
if (curBlock.getNumber() > bestBlockNumber) {
// pruning only on increasing blocks
long pruneBlockNumber = curBlock.getNumber() - pruneBlockCount;
if (pruneBlockNumber >= 0) {
byte[] pruneBlockHash = blockStore.getBlockHashByNumber(pruneBlockNumber);
if (pruneBlockHash != null) {
A0BlockHeader header = blockStore.getBlockByHash(pruneBlockHash).getHeader();
// stateDSPrune.prune(header);
// detailsDS.getStorageDSPrune().prune(header);
stateDSPrune.prune(header);
detailsDS.getStorageDSPrune().prune(header);
}
}
}
bestBlockNumber = curBlock.getNumber();
} */
}

public Trie getWorldState() {
return worldState;
Expand All @@ -543,7 +545,7 @@ public IRepository getSnapshotTo(byte[] root) {
repo.blockStore = blockStore;
repo.cfg = cfg;
repo.stateDatabase = this.stateDatabase;
// repo.stateDSPrune = this.stateDSPrune;
repo.stateDSPrune = this.stateDSPrune;
repo.pruneBlockCount = this.pruneBlockCount;
repo.detailsDS = this.detailsDS;
repo.isSnapshot = true;
Expand Down