Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Merge branches storage #653

Merged
merged 3 commits into from
Oct 2, 2018
Merged

Merge branches storage #653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2dfee0f
Split building methods for using outside Spring Beans
Sep 26, 2018
b7aff96
Extract TrieStore Factory
Sep 26, 2018
2974626
Always save ContractDetails trie to external storage
Sep 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 18 additions & 19 deletions rskj-core/src/main/java/co/rsk/db/ContractDetailsImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import co.rsk.panic.PanicProcessor;
import co.rsk.trie.*;
import org.ethereum.datasource.DataSourcePool;
import org.ethereum.datasource.KeyValueDataSource;
import org.ethereum.db.ByteArrayWrapper;
import org.ethereum.db.ContractDetails;
import org.ethereum.util.RLP;
Expand All @@ -38,7 +37,6 @@
import javax.annotation.Nullable;
import java.util.*;

import static org.ethereum.datasource.DataSourcePool.levelDbByName;
import static org.ethereum.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.ethereum.util.ByteUtil.toHexString;
import static org.ethereum.util.ByteUtil.wrap;
Expand All @@ -50,9 +48,6 @@ public class ContractDetailsImpl implements ContractDetails {
private static final PanicProcessor panicProcessor = new PanicProcessor();
private static final Logger logger = LoggerFactory.getLogger("contractdetails");

private final String databaseDir;
private final int memoryStorageLimit;

private Trie trie;
private byte[] code;
private byte[] address;
Expand All @@ -62,19 +57,21 @@ public class ContractDetailsImpl implements ContractDetails {
private boolean externalStorage;
private boolean closed;
private Set<ByteArrayWrapper> keys = new HashSet<>();
private final TrieStore.Factory trieStoreFactory;
private final int memoryStorageLimit;

public ContractDetailsImpl(byte[] encoded, int memoryStorageLimit, String databaseDir) {
public ContractDetailsImpl(byte[] encoded, TrieStore.Factory trieStoreFactory, int memoryStorageLimit) {
this.trieStoreFactory = trieStoreFactory;
this.memoryStorageLimit = memoryStorageLimit;
this.databaseDir = databaseDir;
decode(encoded);
}

public ContractDetailsImpl(byte[] address, Trie trie, byte[] code, int memoryStorageLimit, String databaseDir) {
public ContractDetailsImpl(byte[] address, Trie trie, byte[] code, TrieStore.Factory trieStoreFactory, int memoryStorageLimit) {
this.address = ByteUtils.clone(address);
this.trie = trie;
this.code = ByteUtils.clone(code);
this.trieStoreFactory = trieStoreFactory;
this.memoryStorageLimit = memoryStorageLimit;
this.databaseDir = databaseDir;
}

@Override
Expand Down Expand Up @@ -190,7 +187,7 @@ public final void decode(byte[] rlpBytes) {

if (this.externalStorage) {
Keccak256 snapshotHash = new Keccak256(rlpStorage.getRLPData());
this.trie = new TrieImpl(new TrieStoreImpl(levelDbByName(getDataSourceName(), databaseDir)), true).getSnapshotTo(snapshotHash);
this.trie = new TrieImpl(trieStoreFactory.newInstance(getDataSourceName()), true).getSnapshotTo(snapshotHash);
} else {
this.trie = TrieImpl.deserialize(rlpStorage.getRLPData());
}
Expand Down Expand Up @@ -229,10 +226,10 @@ public byte[] getEncoded() {
logger.trace("getting contract details as bytes, hash {}, address {}, storage size {}, has external storage {}", this.getStorageHashAsString(), this.getAddressAsString(), this.getStorageSize(), this.hasExternalStorage());

byte[] rlpAddress = RLP.encodeElement(address);
byte[] rlpIsExternalStorage = RLP.encodeByte((byte) (externalStorage ? 1 : 0));
byte[] rlpIsExternalStorage = RLP.encodeByte((byte) 1);

// Serialize the full trie, or only the root hash if external storage is used
byte[] rlpStorage = RLP.encodeElement(externalStorage ? this.trie.getHash().getBytes() : this.trie.serialize());
byte[] rlpStorage = RLP.encodeElement(this.trie.getHash().getBytes());

byte[] rlpCode = RLP.encodeElement(this.code);
byte[] rlpKeys = RLP.encodeSet(this.keys);
Expand Down Expand Up @@ -330,8 +327,7 @@ public synchronized void syncStorage() {
// switching to data source

logger.trace("switching to data source, hash {}, address {}", hashString, addressString);
KeyValueDataSource ds = levelDbByName(this.getDataSourceName(), databaseDir);
TrieStoreImpl newStore = new TrieStoreImpl(ds);
TrieStoreImpl newStore = (TrieStoreImpl) trieStoreFactory.newInstance(this.getDataSourceName());
TrieStoreImpl originalStore = (TrieStoreImpl)((TrieImpl) this.trie).getStore();
newStore.copyFrom(originalStore);
Trie newTrie = newStore.retrieve(this.trie.getHash().getBytes());
Expand Down Expand Up @@ -364,14 +360,18 @@ public synchronized ContractDetails getSnapshotTo(byte[] hash) {

this.trie.save();

ContractDetailsImpl details = new ContractDetailsImpl(this.address, this.trie.getSnapshotTo(new Keccak256(hash)), this.code, this.memoryStorageLimit, this.databaseDir);
ContractDetailsImpl details = new ContractDetailsImpl(this.address,
this.trie.getSnapshotTo(new Keccak256(hash)),
this.code,
this.trieStoreFactory,
this.memoryStorageLimit);
details.keys = new HashSet<>();
details.keys.addAll(this.keys);
details.externalStorage = this.externalStorage;
details.originalExternalStorage = this.originalExternalStorage;

if (this.externalStorage) {
levelDbByName(getDataSourceName(), databaseDir);
DataSourcePool.reserve(getDataSourceName());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pregunta: porque el cambio de levelDBByName*( a DataSourcePool.reserve(). Cual es la diferencia?

}

logger.trace("getting contract details snapshot hash {}, address {}, storage size {}, has external storage {}", details.getStorageHashAsString(), details.getAddressAsString(), details.getStorageSize(), details.hasExternalStorage());
Expand Down Expand Up @@ -401,7 +401,7 @@ private void removeKey(byte[] key) {
}

private void checkExternalStorage() {
this.externalStorage = (keys.size() > memoryStorageLimit) || this.externalStorage;
this.externalStorage = true;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Porque decidieron no usar mas el encoding liviano ?

}

private String getDataSourceName() {
Expand All @@ -428,8 +428,7 @@ private void checkDataSourceIsOpened() {
}

logger.trace("reopening contract details data source");
KeyValueDataSource ds = levelDbByName(this.getDataSourceName(), databaseDir);
TrieStoreImpl newStore = new TrieStoreImpl(ds);
TrieStoreImpl newStore = (TrieStoreImpl) trieStoreFactory.newInstance(getDataSourceName());
Trie newTrie = newStore.retrieve(this.trie.getHash().getBytes());
this.trie = newTrie;
this.closed = false;
Expand Down
38 changes: 23 additions & 15 deletions rskj-core/src/main/java/co/rsk/db/RepositoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,27 +56,35 @@ public class RepositoryImpl implements Repository {

private static final Logger logger = LoggerFactory.getLogger("repository");

private final int memoryStorageLimit;
private final String databaseDir;
private TrieStore store;
private Trie trie;
private DetailsDataStore detailsDataStore;
private boolean closed;
private TrieStore.Factory trieStoreFactory;
private int memoryStorageLimit;

public RepositoryImpl(TrieStore store, int memoryStorageLimit, String databaseDir) {
this(store, new HashMapDB(), memoryStorageLimit, databaseDir);
public RepositoryImpl(TrieStore store, TrieStore.Factory trieStoreFactory, int memoryStorageLimit) {
this(store, new HashMapDB(), trieStoreFactory, memoryStorageLimit);
}

public RepositoryImpl(TrieStore store, KeyValueDataSource detailsDS, int memoryStorageLimit, String databaseDir) {
this(store, new DetailsDataStore(new DatabaseImpl(detailsDS)), memoryStorageLimit, databaseDir);
public RepositoryImpl(
TrieStore store,
KeyValueDataSource detailsDS,
TrieStore.Factory trieStoreFactory,
int memoryStorageLimit) {
this(store, new DetailsDataStore(new DatabaseImpl(detailsDS), trieStoreFactory, memoryStorageLimit), trieStoreFactory, memoryStorageLimit);
}

private RepositoryImpl(TrieStore store, DetailsDataStore detailsDataStore, int memoryStorageLimit, String databaseDir) {
private RepositoryImpl(
TrieStore store,
DetailsDataStore detailsDataStore,
TrieStore.Factory trieStoreFactory,
int memoryStorageLimit) {
this.store = store;
this.trie = new TrieImpl(store, true);
this.detailsDataStore = detailsDataStore;
this.trieStoreFactory = trieStoreFactory;
this.memoryStorageLimit = memoryStorageLimit;
this.databaseDir = databaseDir;
}

@Override
Expand All @@ -87,8 +95,8 @@ public synchronized AccountState createAccount(RskAddress addr) {
null,
new TrieImpl(new TrieStoreImpl(new HashMapDB()), true),
null,
memoryStorageLimit,
databaseDir
trieStoreFactory,
memoryStorageLimit
));
return accountState;
}
Expand Down Expand Up @@ -152,7 +160,7 @@ public synchronized ContractDetails getContractDetails(RskAddress addr) {
storageRoot = getAccountState(addr).getStateRoot();
}

ContractDetails details = detailsDataStore.get(addr, memoryStorageLimit, databaseDir);
ContractDetails details = detailsDataStore.get(addr);
if (details != null) {
details = details.getSnapshotTo(storageRoot);
}
Expand Down Expand Up @@ -274,7 +282,7 @@ public synchronized void dumpState(Block block, long gasUsed, int txNumber, byte

@Override
public synchronized Repository startTracking() {
return new RepositoryTrack(this);
return new RepositoryTrack(this, trieStoreFactory, memoryStorageLimit);
}

@Override
Expand Down Expand Up @@ -349,8 +357,8 @@ public synchronized void updateBatch(Map<RskAddress, AccountState> stateCache,
null,
new TrieImpl(new TrieStoreImpl(new HashMapDB()), true),
null,
memoryStorageLimit,
databaseDir
trieStoreFactory,
memoryStorageLimit
);
originalContractDetails.setAddress(addr.getBytes());
contractDetailsCache.setOriginalContractDetails(originalContractDetails);
Expand Down Expand Up @@ -404,7 +412,7 @@ public synchronized void loadAccount(RskAddress addr,

@Override
public synchronized Repository getSnapshotTo(byte[] root) {
RepositoryImpl snapshotRepository = new RepositoryImpl(this.store, this.detailsDataStore, memoryStorageLimit, databaseDir);
RepositoryImpl snapshotRepository = new RepositoryImpl(this.store, this.detailsDataStore, this.trieStoreFactory, this.memoryStorageLimit);
snapshotRepository.syncToRoot(root);
return snapshotRepository;
}
Expand Down
4 changes: 4 additions & 0 deletions rskj-core/src/main/java/co/rsk/trie/TrieStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,8 @@ public interface TrieStore {
byte[] serialize();

byte[] retrieveValue(byte[] hash);

interface Factory {
TrieStore newInstance(String name);
}
}
17 changes: 12 additions & 5 deletions rskj-core/src/main/java/org/ethereum/config/CommonConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.util.List;

import static java.util.Arrays.asList;
import static org.ethereum.datasource.DataSourcePool.levelDbByName;

@Configuration
@ComponentScan(
Expand All @@ -56,15 +57,21 @@ public Repository repository(RskSystemProperties config) {
FileUtil.recursiveDelete(databaseDir);
logger.info("Database reset done");
}
return buildRepository(databaseDir, config.detailsInMemoryStorageLimit());
}

KeyValueDataSource ds = makeDataSource(config, "state");
KeyValueDataSource detailsDS = makeDataSource(config, "details");
public Repository buildRepository(String databaseDir, int memoryStorageLimit) {
KeyValueDataSource ds = makeDataSource("state", databaseDir);
KeyValueDataSource detailsDS = makeDataSource("details", databaseDir);

return new RepositoryImpl(new TrieStoreImpl(ds), detailsDS, config.detailsInMemoryStorageLimit(), config.databaseDir());
return new RepositoryImpl(new TrieStoreImpl(ds), detailsDS,
name -> new TrieStoreImpl(levelDbByName(name, databaseDir)),
memoryStorageLimit
);
}

private KeyValueDataSource makeDataSource(RskSystemProperties config, String name) {
KeyValueDataSource ds = new LevelDbDataSource(name, config.databaseDir());
private KeyValueDataSource makeDataSource(String name, String databaseDir) {
KeyValueDataSource ds = new LevelDbDataSource(name, databaseDir);
ds.init();
return ds;
}
Expand Down
18 changes: 11 additions & 7 deletions rskj-core/src/main/java/org/ethereum/config/DefaultConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,11 @@ public class DefaultConfig {

@Bean
public BlockStore blockStore(RskSystemProperties config) {
String database = config.databaseDir();
return buildBlockStore(config.databaseDir());
}

File blockIndexDirectory = new File(database + "/blocks/");
public BlockStore buildBlockStore(String databaseDir) {
File blockIndexDirectory = new File(databaseDir + "/blocks/");
File dbFile = new File(blockIndexDirectory, "index");
if (!blockIndexDirectory.exists()) {
boolean mkdirsSuccess = blockIndexDirectory.mkdirs();
Expand All @@ -89,17 +91,19 @@ public BlockStore blockStore(RskSystemProperties config) {
.counterEnable()
.makeOrGet();

KeyValueDataSource blocksDB = new LevelDbDataSource("blocks", database);
KeyValueDataSource blocksDB = new LevelDbDataSource("blocks", databaseDir);
blocksDB.init();

IndexedBlockStore indexedBlockStore = new IndexedBlockStore(indexMap, blocksDB, indexDB);

return indexedBlockStore;
return new IndexedBlockStore(indexMap, blocksDB, indexDB);
}

@Bean
public ReceiptStore receiptStore(RskSystemProperties config) {
KeyValueDataSource ds = new LevelDbDataSource("receipts", config.databaseDir());
return buildReceiptStore(config.databaseDir());
}

public ReceiptStore buildReceiptStore(String databaseDir) {
KeyValueDataSource ds = new LevelDbDataSource("receipts", databaseDir);
ds.init();
return new ReceiptStoreImpl(ds);
}
Expand Down
4 changes: 2 additions & 2 deletions rskj-core/src/main/java/org/ethereum/core/genesis/ContractDetailsMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ public ContractDetails mapFromContract(Contract contract) {
null,
new TrieImpl(new TrieStoreImpl(new HashMapDB()), true),
null,
config.detailsInMemoryStorageLimit(),
config.databaseDir()
null,
config.detailsInMemoryStorageLimit()
);

if (contract.getCode()!=null) {
Expand Down
9 changes: 9 additions & 0 deletions rskj-core/src/main/java/org/ethereum/datasource/DataSourcePool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,15 @@ public static KeyValueDataSource levelDbByName(String name, String databaseDir)
return (KeyValueDataSource) result.getDataSource();
}

public static void reserve(String name) {
DataSourceEx dataSourceEx = pool.get(name);
if (dataSourceEx != null) {
synchronized (dataSourceEx) {
dataSourceEx.reserve();
}
}
}

public static void closeDataSource(String name){
DataSourceEx dataSourceEx = pool.get(name);

Expand Down
18 changes: 13 additions & 5 deletions rskj-core/src/main/java/org/ethereum/db/DetailsDataStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import co.rsk.core.RskAddress;
import co.rsk.db.ContractDetailsImpl;
import co.rsk.trie.TrieStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -43,12 +44,16 @@ public class DetailsDataStore {
private final Set<RskAddress> removes = new HashSet<>();

private final DatabaseImpl db;
private final int memoryStorageLimit;
private TrieStore.Factory trieStoreFactory;

public DetailsDataStore(DatabaseImpl db) {
public DetailsDataStore(DatabaseImpl db, TrieStore.Factory trieStoreFactory, int memoryStorageLimit) {
this.db = db;
this.trieStoreFactory = trieStoreFactory;
this.memoryStorageLimit = memoryStorageLimit;
}

public synchronized ContractDetails get(RskAddress addr, int memoryStorageLimit, String databaseDir) {
public synchronized ContractDetails get(RskAddress addr) {
ContractDetails details = cache.get(addr);

if (details == null) {
Expand All @@ -61,7 +66,7 @@ public synchronized ContractDetails get(RskAddress addr, int memoryStorageLimit,
return null;
}

details = createContractDetails(data, memoryStorageLimit, databaseDir);
details = createContractDetails(data, trieStoreFactory, memoryStorageLimit);
cache.put(addr, details);

float out = ((float) data.length) / 1048576;
Expand All @@ -74,8 +79,11 @@ public synchronized ContractDetails get(RskAddress addr, int memoryStorageLimit,
return details;
}

protected ContractDetails createContractDetails(byte[] data, int memoryStorageLimit, String databaseDir) {
return new ContractDetailsImpl(data, memoryStorageLimit, databaseDir);
protected ContractDetails createContractDetails(
byte[] data,
TrieStore.Factory trieStoreFactory,
int memoryStorageLimit) {
return new ContractDetailsImpl(data, trieStoreFactory, memoryStorageLimit);
}

public synchronized void update(RskAddress addr, ContractDetails contractDetails) {
Expand Down
Loading