Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor and test journal pruning class #482

Merged
merged 10 commits into from Jun 1, 2018
Merged

Refactor and test journal pruning class #482

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
601e905
refactored JournalPruneDataSource to be independent of the block and …
AlexandraRoatis May 11, 2018
e92d842
replacing synchronized methods with read-write locks
AlexandraRoatis May 14, 2018
0fb29b9
disabling locking on the state database since the JournalPruneDS is l…
AlexandraRoatis May 14, 2018
8aa1133
moving the JournalPruneDS test class in the correct folder
AlexandraRoatis May 14, 2018
ca0c44b
unit tests for conformance to interface
AlexandraRoatis May 14, 2018
d10df63
concurrency tests for JournalPruneDS
AlexandraRoatis May 14, 2018
3205e07
unit tests for pruning functionality
AlexandraRoatis May 14, 2018
0f78cc8
removed dependency on modDbImpl
AlexandraRoatis May 14, 2018
527f382
applying review comments
AlexandraRoatis May 17, 2018
606b1e3
bugfix: incorrect check for presence in the data source
AlexandraRoatis May 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
189 changes: 98 additions & 91 deletions modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java
View file
Open in desktop
@@ -1,4 +1,4 @@
/*******************************************************************************
/* ******************************************************************************
* Copyright (c) 2017-2018 Aion foundation.
*
* This file is part of the aion network project.
Expand All @@ -19,11 +19,15 @@
*
* Contributors:
* Aion foundation.
*
******************************************************************************/

package org.aion.zero.impl.db;

import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH;

import java.io.File;
import java.math.BigInteger;
import java.util.*;
import org.aion.base.db.*;
import org.aion.base.type.Address;
import org.aion.base.util.Hex;
Expand All @@ -42,28 +46,19 @@
import org.aion.zero.types.AionTransaction;
import org.aion.zero.types.AionTxReceipt;

import java.io.File;
import java.math.BigInteger;
import java.util.*;

import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH;

/**
* Has direct database connection.
*/
public class AionRepositoryImpl extends AbstractRepository<AionBlock, A0BlockHeader, AionBlockStore> {
/** Has direct database connection. */
public class AionRepositoryImpl
extends AbstractRepository<AionBlock, A0BlockHeader, AionBlockStore> {

private TransactionStore<AionTransaction, AionTxReceipt, AionTxInfo> transactionStore;

/**
* used by getSnapShotTo
*
* @ATTENTION: when do snap shot, another instance will be created. Make
* sure it is used only by getSnapShotTo
* <p>@ATTENTION: when do snap shot, another instance will be created. Make sure it is used only
* by getSnapShotTo
*/
protected AionRepositoryImpl() {
}
protected AionRepositoryImpl() {}

protected AionRepositoryImpl(IRepositoryConfig repoConfig) {
this.cfg = repoConfig;
Expand All @@ -74,16 +69,24 @@ private static class AionRepositoryImplHolder {
// configuration
private static CfgAion config = CfgAion.inst();
// repository singleton instance
private final static AionRepositoryImpl inst = new AionRepositoryImpl(
new RepositoryConfig(new File(config.getBasePath(), config.getDb().getPath()).getAbsolutePath(),
config.getDb().getPrune() > 0 ?
// if the value is smaller than backward step
// there is the risk of importing state-less blocks after reboot
(128 > config.getDb().getPrune() ? 128 : config.getDb().getPrune()) :
// negative value => pruning disabled
config.getDb().getPrune(),
ContractDetailsAion.getInstance(),
config.getDb()));
private static final AionRepositoryImpl inst =
new AionRepositoryImpl(
new RepositoryConfig(
new File(config.getBasePath(), config.getDb().getPath())
.getAbsolutePath(),
config.getDb().getPrune() > 0
?
// if the value is smaller than backward step
// there is the risk of importing state-less blocks after
// reboot
(128 > config.getDb().getPrune()
? 128
: config.getDb().getPrune())
:
// negative value => pruning disabled
config.getDb().getPrune(),
ContractDetailsAion.getInstance(),
config.getDb()));
}

public static AionRepositoryImpl inst() {
Expand All @@ -99,8 +102,9 @@ private void init() {
initializeDatabasesAndCaches();

// Setup the cache for transaction data source.
this.transactionStore = new TransactionStore<>(transactionDatabase,
AionTransactionStoreSerializer.serializer);
this.transactionStore =
new TransactionStore<>(
transactionDatabase, AionTransactionStoreSerializer.serializer);

// Setup block store.
this.blockStore = new AionBlockStore(indexDatabase, blockDatabase, checkIntegrity);
Expand All @@ -112,9 +116,7 @@ private void init() {
}
}

/**
* @implNote The transaction store is not locked within the repository implementation.
*/
/** @implNote The transaction store is not locked within the repository implementation. */
public TransactionStore<AionTransaction, AionTxReceipt, AionTxInfo> getTransactionStore() {
return this.transactionStore;
}
Expand All @@ -124,7 +126,8 @@ private Trie createStateTrie() {
}

@Override
public void updateBatch(Map<Address, AccountState> stateCache,
public void updateBatch(
Map<Address, AccountState> stateCache,
Map<Address, IContractDetails<DataWord>> detailsCache) {
rwLock.writeLock().lock();

Expand Down Expand Up @@ -153,25 +156,28 @@ public void updateBatch(Map<Address, AccountState> stateCache,
updateAccountState(address, accountState);

if (LOG.isTraceEnabled()) {
LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]",
Hex.toHexString(address.toBytes()),
accountState.getNonce(),
accountState.getBalance(),
Hex.toHexString(contractDetails.getStorageHash()));
LOG.trace(
"update: [{}],nonce: [{}] balance: [{}] [{}]",
Hex.toHexString(address.toBytes()),
accountState.getNonce(),
accountState.getBalance(),
Hex.toHexString(contractDetails.getStorageHash()));
}
}
continue;
}

ContractDetailsCacheImpl contractDetailsCache = (ContractDetailsCacheImpl) contractDetails;
ContractDetailsCacheImpl contractDetailsCache =
(ContractDetailsCacheImpl) contractDetails;
if (contractDetailsCache.origContract == null) {
contractDetailsCache.origContract = this.cfg.contractDetailsImpl();

try {
contractDetailsCache.origContract.setAddress(address);
} catch (Exception e) {
e.printStackTrace();
LOG.error("contractDetailsCache setAddress exception [{}]", e.toString());
LOG.error(
"contractDetailsCache setAddress exception [{}]", e.toString());
}

contractDetailsCache.commit();
Expand All @@ -188,11 +194,12 @@ public void updateBatch(Map<Address, AccountState> stateCache,
updateAccountState(address, accountState);

if (LOG.isTraceEnabled()) {
LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]",
Hex.toHexString(address.toBytes()),
accountState.getNonce(),
accountState.getBalance(),
Hex.toHexString(contractDetails.getStorageHash()));
LOG.trace(
"update: [{}],nonce: [{}] balance: [{}] [{}]",
Hex.toHexString(address.toBytes()),
accountState.getNonce(),
accountState.getBalance(),
Hex.toHexString(contractDetails.getStorageHash()));
}
}
}
Expand All @@ -205,10 +212,9 @@ public void updateBatch(Map<Address, AccountState> stateCache,
}
}

/**
* @implNote The method calling this method must handle the locking.
*/
private void updateContractDetails(final Address address, final IContractDetails<DataWord> contractDetails) {
/** @implNote The method calling this method must handle the locking. */
private void updateContractDetails(
final Address address, final IContractDetails<DataWord> contractDetails) {
// locked by calling method
detailsDS.update(address, contractDetails);
}
Expand Down Expand Up @@ -369,20 +375,18 @@ public BigInteger getNonce(Address address) {
return (account == null) ? BigInteger.ZERO : account.getNonce();
}

/**
* @implNote The method calling this method must handle the locking.
*/
/** @implNote The method calling this method must handle the locking. */
private void updateAccountState(Address address, AccountState accountState) {
// locked by calling method
worldState.update(address.toBytes(), accountState.getEncoded());
}

/**
* @inheritDoc
* @implNote Any other method calling this can rely on the fact that
* the contract details returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}.
* Since this querying method it locked, the methods calling it
* <b>may not need to be locked or synchronized</b>, depending on the specific use case.
* @implNote Any other method calling this can rely on the fact that the contract details
* returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}.
* Since this querying method it locked, the methods calling it <b>may not need to be locked
* or synchronized</b>, depending on the specific use case.
*/
@Override
public IContractDetails<DataWord> getContractDetails(Address address) {
Expand Down Expand Up @@ -423,10 +427,9 @@ public boolean hasContractDetails(Address address) {

/**
* @inheritDoc
* @implNote Any other method calling this can rely on the fact that
* the account state returned is a newly created object.
* Since this querying method it locked, the methods calling it
* <b>may not need to be locked or synchronized</b>, depending on the specific use case.
* @implNote Any other method calling this can rely on the fact that the account state returned
* is a newly created object. Since this querying method it locked, the methods calling it
* <b>may not need to be locked or synchronized</b>, depending on the specific use case.
*/
@Override
public AccountState getAccountState(Address address) {
Expand All @@ -439,7 +442,8 @@ public AccountState getAccountState(Address address) {

if (accountData.length != 0) {
result = new AccountState(accountData);
LOG.debug("New AccountSate [{}], State [{}]", address.toString(), result.toString());
LOG.debug(
"New AccountSate [{}], State [{}]", address.toString(), result.toString());
}
return result;
} finally {
Expand All @@ -453,11 +457,13 @@ public boolean hasAccountState(Address address) {
}

/**
* @implNote The loaded objects are fresh copies of the original account
* state and contract details.
* @implNote The loaded objects are fresh copies of the original account state and contract
* details.
*/
@Override
public void loadAccountState(Address address, Map<Address, AccountState> cacheAccounts,
public void loadAccountState(
Address address,
Map<Address, AccountState> cacheAccounts,
Map<Address, IContractDetails<DataWord>> cacheDetails) {

AccountState account = getAccountState(address);
Expand Down Expand Up @@ -507,8 +513,10 @@ public void commitBlock(A0BlockHeader blockHeader) {
detailsDS.syncLargeStorage();

if (pruneBlockCount > 0) {
stateDSPrune.storeBlockChanges(blockHeader);
detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader);
stateDSPrune.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber());
detailsDS
.getStorageDSPrune()
.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber());
pruneBlocks(blockHeader);
}
} finally {
Expand All @@ -524,8 +532,8 @@ private void pruneBlocks(A0BlockHeader curBlock) {
byte[] pruneBlockHash = blockStore.getBlockHashByNumber(pruneBlockNumber);
if (pruneBlockHash != null) {
A0BlockHeader header = blockStore.getBlockByHash(pruneBlockHash).getHeader();
stateDSPrune.prune(header);
detailsDS.getStorageDSPrune().prune(header);
stateDSPrune.prune(header.getHash(), header.getNumber());
detailsDS.getStorageDSPrune().prune(header.getHash(), header.getNumber());
}
}
}
Expand Down Expand Up @@ -597,10 +605,7 @@ public void removeTxBatch(Set<byte[]> clearTxSet, boolean isPool) {
}
}

/**
* This function cannot for any reason fail, otherwise we may have dangling
* file IO locks
*/
/** This function cannot for any reason fail, otherwise we may have dangling file IO locks */
@Override
public void close() {
rwLock.writeLock().lock();
Expand Down Expand Up @@ -662,20 +667,20 @@ public void close() {
pendingTxCacheDatabase = null;
}
} catch (Exception e) {
LOGGEN.error("Exception occurred while closing the pendingTxCacheDatabase store.", e);
LOGGEN.error(
"Exception occurred while closing the pendingTxCacheDatabase store.", e);
}
} finally {
rwLock.writeLock().unlock();
}
}

/**
* Retrieves the underlying state database that sits below all caches. This
* is usually provided by {@link org.aion.db.impl.leveldb.LevelDB} or
* {@link org.aion.db.impl.leveldb.LevelDB}.
* <p>
* Note that referencing the state database directly is unsafe, and should
* only be used for debugging and testing purposes.
* Retrieves the underlying state database that sits below all caches. This is usually provided
* by {@link org.aion.db.impl.leveldb.LevelDB} or {@link org.aion.db.impl.leveldb.LevelDB}.
*
* <p>Note that referencing the state database directly is unsafe, and should only be used for
* debugging and testing purposes.
*
* @return
*/
Expand All @@ -684,30 +689,32 @@ public IByteArrayKeyValueDatabase getStateDatabase() {
}

/**
* Retrieves the underlying details database that sits below all caches.
* This is usually provided by {@link org.aion.db.impl.mockdb.MockDB}
* or {@link org.aion.db.impl.mockdb.MockDB}.
* <p>
* Note that referencing the state database directly is unsafe, and should
* only be used for debugging and testing purposes.
* Retrieves the underlying details database that sits below all caches. This is usually
* provided by {@link org.aion.db.impl.mockdb.MockDB} or {@link org.aion.db.impl.mockdb.MockDB}.
*
* <p>Note that referencing the state database directly is unsafe, and should only be used for
* debugging and testing purposes.
*
* @return
*/
public IByteArrayKeyValueDatabase getDetailsDatabase() {
return this.detailsDatabase;
}

/**
* For testing.
*/
/** For testing. */
public IByteArrayKeyValueDatabase getIndexDatabase() {
return this.indexDatabase;
}

@Override
public String toString() {
return "AionRepositoryImpl{ identityHashCode=" + System.identityHashCode(this) + ", " + //
"databaseGroupSize=" + (databaseGroup == null ? 0 : databaseGroup.size()) + '}';
return "AionRepositoryImpl{ identityHashCode="
+ System.identityHashCode(this)
+ ", "
+ //
"databaseGroupSize="
+ (databaseGroup == null ? 0 : databaseGroup.size())
+ '}';
}

@Override
Expand Down