Skip to content

Commit

Permalink
Merge pull request #482 from aionnetwork/db-prune-refactor
Browse files Browse the repository at this point in the history 
Refactor and test journal pruning class
  • Loading branch information
@AionJayT
AionJayT committed Jun 1, 2018
2 parents 46fccd6 + 606b1e3 commit 8326973
Show file tree
Hide file tree
Showing 6 changed files with 1,994 additions and 824 deletions.
189 changes: 98 additions & 91 deletions modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java
View file
@@ -1,4 +1,4 @@
/*******************************************************************************
/* ******************************************************************************
* Copyright (c) 2017-2018 Aion foundation.
*
* This file is part of the aion network project.
Expand All @@ -19,11 +19,15 @@
*
* Contributors:
* Aion foundation.
*
******************************************************************************/

package org.aion.zero.impl.db;

import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH;

import java.io.File;
import java.math.BigInteger;
import java.util.*;
import org.aion.base.db.*;
import org.aion.base.type.Address;
import org.aion.base.util.Hex;
Expand All @@ -42,28 +46,19 @@
import org.aion.zero.types.AionTransaction;
import org.aion.zero.types.AionTxReceipt;

import java.io.File;
import java.math.BigInteger;
import java.util.*;

import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH;

/**
* Has direct database connection.
*/
public class AionRepositoryImpl extends AbstractRepository<AionBlock, A0BlockHeader, AionBlockStore> {
/** Has direct database connection. */
public class AionRepositoryImpl
extends AbstractRepository<AionBlock, A0BlockHeader, AionBlockStore> {

private TransactionStore<AionTransaction, AionTxReceipt, AionTxInfo> transactionStore;

/**
* used by getSnapShotTo
*
* @ATTENTION: when do snap shot, another instance will be created. Make
* sure it is used only by getSnapShotTo
* <p>@ATTENTION: when do snap shot, another instance will be created. Make sure it is used only
* by getSnapShotTo
*/
protected AionRepositoryImpl() {
}
protected AionRepositoryImpl() {}

protected AionRepositoryImpl(IRepositoryConfig repoConfig) {
this.cfg = repoConfig;
Expand All @@ -74,16 +69,24 @@ private static class AionRepositoryImplHolder {
// configuration
private static CfgAion config = CfgAion.inst();
// repository singleton instance
private final static AionRepositoryImpl inst = new AionRepositoryImpl(
new RepositoryConfig(new File(config.getBasePath(), config.getDb().getPath()).getAbsolutePath(),
config.getDb().getPrune() > 0 ?
// if the value is smaller than backward step
// there is the risk of importing state-less blocks after reboot
(128 > config.getDb().getPrune() ? 128 : config.getDb().getPrune()) :
// negative value => pruning disabled
config.getDb().getPrune(),
ContractDetailsAion.getInstance(),
config.getDb()));
private static final AionRepositoryImpl inst =
new AionRepositoryImpl(
new RepositoryConfig(
new File(config.getBasePath(), config.getDb().getPath())
.getAbsolutePath(),
config.getDb().getPrune() > 0
?
// if the value is smaller than backward step
// there is the risk of importing state-less blocks after
// reboot
(128 > config.getDb().getPrune()
? 128
: config.getDb().getPrune())
:
// negative value => pruning disabled
config.getDb().getPrune(),
ContractDetailsAion.getInstance(),
config.getDb()));
}

public static AionRepositoryImpl inst() {
Expand All @@ -99,8 +102,9 @@ private void init() {
initializeDatabasesAndCaches();

// Setup the cache for transaction data source.
this.transactionStore = new TransactionStore<>(transactionDatabase,
AionTransactionStoreSerializer.serializer);
this.transactionStore =
new TransactionStore<>(
transactionDatabase, AionTransactionStoreSerializer.serializer);

// Setup block store.
this.blockStore = new AionBlockStore(indexDatabase, blockDatabase, checkIntegrity);
Expand All @@ -112,9 +116,7 @@ private void init() {
}
}

/**
* @implNote The transaction store is not locked within the repository implementation.
*/
/** @implNote The transaction store is not locked within the repository implementation. */
public TransactionStore<AionTransaction, AionTxReceipt, AionTxInfo> getTransactionStore() {
return this.transactionStore;
}
Expand All @@ -124,7 +126,8 @@ private Trie createStateTrie() {
}

@Override
public void updateBatch(Map<Address, AccountState> stateCache,
public void updateBatch(
Map<Address, AccountState> stateCache,
Map<Address, IContractDetails<DataWord>> detailsCache) {
rwLock.writeLock().lock();

Expand Down Expand Up @@ -153,25 +156,28 @@ public void updateBatch(Map<Address, AccountState> stateCache,
updateAccountState(address, accountState);

if (LOG.isTraceEnabled()) {
LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]",
Hex.toHexString(address.toBytes()),
accountState.getNonce(),
accountState.getBalance(),
Hex.toHexString(contractDetails.getStorageHash()));
LOG.trace(
"update: [{}],nonce: [{}] balance: [{}] [{}]",
Hex.toHexString(address.toBytes()),
accountState.getNonce(),
accountState.getBalance(),
Hex.toHexString(contractDetails.getStorageHash()));
}
}
continue;
}

ContractDetailsCacheImpl contractDetailsCache = (ContractDetailsCacheImpl) contractDetails;
ContractDetailsCacheImpl contractDetailsCache =
(ContractDetailsCacheImpl) contractDetails;
if (contractDetailsCache.origContract == null) {
contractDetailsCache.origContract = this.cfg.contractDetailsImpl();

try {
contractDetailsCache.origContract.setAddress(address);
} catch (Exception e) {
e.printStackTrace();
LOG.error("contractDetailsCache setAddress exception [{}]", e.toString());
LOG.error(
"contractDetailsCache setAddress exception [{}]", e.toString());
}

contractDetailsCache.commit();
Expand All @@ -188,11 +194,12 @@ public void updateBatch(Map<Address, AccountState> stateCache,
updateAccountState(address, accountState);

if (LOG.isTraceEnabled()) {
LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]",
Hex.toHexString(address.toBytes()),
accountState.getNonce(),
accountState.getBalance(),
Hex.toHexString(contractDetails.getStorageHash()));
LOG.trace(
"update: [{}],nonce: [{}] balance: [{}] [{}]",
Hex.toHexString(address.toBytes()),
accountState.getNonce(),
accountState.getBalance(),
Hex.toHexString(contractDetails.getStorageHash()));
}
}
}
Expand All @@ -205,10 +212,9 @@ public void updateBatch(Map<Address, AccountState> stateCache,
}
}

/**
* @implNote The method calling this method must handle the locking.
*/
private void updateContractDetails(final Address address, final IContractDetails<DataWord> contractDetails) {
/** @implNote The method calling this method must handle the locking. */
private void updateContractDetails(
final Address address, final IContractDetails<DataWord> contractDetails) {
// locked by calling method
detailsDS.update(address, contractDetails);
}
Expand Down Expand Up @@ -369,20 +375,18 @@ public BigInteger getNonce(Address address) {
return (account == null) ? BigInteger.ZERO : account.getNonce();
}

/**
* @implNote The method calling this method must handle the locking.
*/
/** @implNote The method calling this method must handle the locking. */
private void updateAccountState(Address address, AccountState accountState) {
// locked by calling method
worldState.update(address.toBytes(), accountState.getEncoded());
}

/**
* @inheritDoc
* @implNote Any other method calling this can rely on the fact that
* the contract details returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}.
* Since this querying method it locked, the methods calling it
* <b>may not need to be locked or synchronized</b>, depending on the specific use case.
* @implNote Any other method calling this can rely on the fact that the contract details
* returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}.
* Since this querying method it locked, the methods calling it <b>may not need to be locked
* or synchronized</b>, depending on the specific use case.
*/
@Override
public IContractDetails<DataWord> getContractDetails(Address address) {
Expand Down Expand Up @@ -423,10 +427,9 @@ public boolean hasContractDetails(Address address) {

/**
* @inheritDoc
* @implNote Any other method calling this can rely on the fact that
* the account state returned is a newly created object.
* Since this querying method it locked, the methods calling it
* <b>may not need to be locked or synchronized</b>, depending on the specific use case.
* @implNote Any other method calling this can rely on the fact that the account state returned
* is a newly created object. Since this querying method it locked, the methods calling it
* <b>may not need to be locked or synchronized</b>, depending on the specific use case.
*/
@Override
public AccountState getAccountState(Address address) {
Expand All @@ -439,7 +442,8 @@ public AccountState getAccountState(Address address) {

if (accountData.length != 0) {
result = new AccountState(accountData);
LOG.debug("New AccountSate [{}], State [{}]", address.toString(), result.toString());
LOG.debug(
"New AccountSate [{}], State [{}]", address.toString(), result.toString());
}
return result;
} finally {
Expand All @@ -453,11 +457,13 @@ public boolean hasAccountState(Address address) {
}

/**
* @implNote The loaded objects are fresh copies of the original account
* state and contract details.
* @implNote The loaded objects are fresh copies of the original account state and contract
* details.
*/
@Override
public void loadAccountState(Address address, Map<Address, AccountState> cacheAccounts,
public void loadAccountState(
Address address,
Map<Address, AccountState> cacheAccounts,
Map<Address, IContractDetails<DataWord>> cacheDetails) {

AccountState account = getAccountState(address);
Expand Down Expand Up @@ -507,8 +513,10 @@ public void commitBlock(A0BlockHeader blockHeader) {
detailsDS.syncLargeStorage();

if (pruneBlockCount > 0) {
stateDSPrune.storeBlockChanges(blockHeader);
detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader);
stateDSPrune.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber());
detailsDS
.getStorageDSPrune()
.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber());
pruneBlocks(blockHeader);
}
} finally {
Expand All @@ -524,8 +532,8 @@ private void pruneBlocks(A0BlockHeader curBlock) {
byte[] pruneBlockHash = blockStore.getBlockHashByNumber(pruneBlockNumber);
if (pruneBlockHash != null) {
A0BlockHeader header = blockStore.getBlockByHash(pruneBlockHash).getHeader();
stateDSPrune.prune(header);
detailsDS.getStorageDSPrune().prune(header);
stateDSPrune.prune(header.getHash(), header.getNumber());
detailsDS.getStorageDSPrune().prune(header.getHash(), header.getNumber());
}
}
}
Expand Down Expand Up @@ -597,10 +605,7 @@ public void removeTxBatch(Set<byte[]> clearTxSet, boolean isPool) {
}
}

/**
* This function cannot for any reason fail, otherwise we may have dangling
* file IO locks
*/
/** This function cannot for any reason fail, otherwise we may have dangling file IO locks */
@Override
public void close() {
rwLock.writeLock().lock();
Expand Down Expand Up @@ -662,20 +667,20 @@ public void close() {
pendingTxCacheDatabase = null;
}
} catch (Exception e) {
LOGGEN.error("Exception occurred while closing the pendingTxCacheDatabase store.", e);
LOGGEN.error(
"Exception occurred while closing the pendingTxCacheDatabase store.", e);
}
} finally {
rwLock.writeLock().unlock();
}
}

/**
* Retrieves the underlying state database that sits below all caches. This
* is usually provided by {@link org.aion.db.impl.leveldb.LevelDB} or
* {@link org.aion.db.impl.leveldb.LevelDB}.
* <p>
* Note that referencing the state database directly is unsafe, and should
* only be used for debugging and testing purposes.
* Retrieves the underlying state database that sits below all caches. This is usually provided
* by {@link org.aion.db.impl.leveldb.LevelDB} or {@link org.aion.db.impl.leveldb.LevelDB}.
*
* <p>Note that referencing the state database directly is unsafe, and should only be used for
* debugging and testing purposes.
*
* @return
*/
Expand All @@ -684,30 +689,32 @@ public IByteArrayKeyValueDatabase getStateDatabase() {
}

/**
* Retrieves the underlying details database that sits below all caches.
* This is usually provided by {@link org.aion.db.impl.mockdb.MockDB}
* or {@link org.aion.db.impl.mockdb.MockDB}.
* <p>
* Note that referencing the state database directly is unsafe, and should
* only be used for debugging and testing purposes.
* Retrieves the underlying details database that sits below all caches. This is usually
* provided by {@link org.aion.db.impl.mockdb.MockDB} or {@link org.aion.db.impl.mockdb.MockDB}.
*
* <p>Note that referencing the state database directly is unsafe, and should only be used for
* debugging and testing purposes.
*
* @return
*/
public IByteArrayKeyValueDatabase getDetailsDatabase() {
return this.detailsDatabase;
}

/**
* For testing.
*/
/** For testing. */
public IByteArrayKeyValueDatabase getIndexDatabase() {
return this.indexDatabase;
}

@Override
public String toString() {
return "AionRepositoryImpl{ identityHashCode=" + System.identityHashCode(this) + ", " + //
"databaseGroupSize=" + (databaseGroup == null ? 0 : databaseGroup.size()) + '}';
return "AionRepositoryImpl{ identityHashCode="
+ System.identityHashCode(this)
+ ", "
+ //
"databaseGroupSize="
+ (databaseGroup == null ? 0 : databaseGroup.size())
+ '}';
}

@Override
Expand Down

0 comments on commit 8326973

Please sign in to comment.