Skip to content

Commit

Permalink
Working version of StorageDictionary
Browse files Browse the repository at this point in the history
  • Loading branch information
@Nashatyrev
Nashatyrev committed Feb 20, 2016
1 parent cc551ce commit 7d9d131
Show file tree
Hide file tree
Showing 3 changed files with 145 additions and 64 deletions.
2 changes: 1 addition & 1 deletion ethereumj-core/src/main/java/org/ethereum/datasource/XorDataSource.java
View file
Expand Up @@ -20,7 +20,7 @@ public XorDataSource(KeyValueDataSource source, byte[] subKey) {
}

private byte[] convertKey(byte[] key) {
return ByteUtil.xor(key, subKey);
return ByteUtil.xorAlignRight(key, subKey);
}

@Override
Expand Down
93 changes: 71 additions & 22 deletions ethereumj-core/src/main/java/org/ethereum/db/StorageDictionary.java
View file
Expand Up @@ -89,6 +89,9 @@
*/
public class StorageDictionary {

private static final int MAX_CHILDREN_TO_SORT = 100;
private static final boolean SORT_MAP_KEYS = false;

public enum Type {
Root,
StorageIndex, // top-level Contract field index
Expand Down Expand Up @@ -180,16 +183,48 @@ public PathElement addChild(PathElement newChild) {
return existingChild;
}

if (childrenCount == 0) {
firstChildHash = newChild.getHash();
if (childrenCount > MAX_CHILDREN_TO_SORT || (!SORT_MAP_KEYS && newChild.type == Type.MapKey)) {
// no more sorting just add to the end
insertChild(getLastChild(), newChild);
} else {
PathElement lastChild = getLastChild();
lastChild.nextSiblingHash = newChild.getHash();
lastChild.invalidate();
Iterator<PathElement> chIt = getChildrenIterator();
PathElement insertAfter = null;
while(chIt.hasNext()) {
PathElement next = chIt.next();
if (newChild.compareTo(next) < 0) {
break;
}
insertAfter = next;
}
insertChild(insertAfter, newChild);
}
lastChildHash = newChild.getHash();
newChild.parentHash = this.getHash();

return newChild;
}

private boolean isMapping() {
return getFirstChild().type == Type.MapKey;
}

public PathElement insertChild(PathElement insertAfter, PathElement newChild) {
if (insertAfter == null) {
// first element
newChild.nextSiblingHash = firstChildHash;
firstChildHash = newChild.storageKey;
if (childrenCount == 0) {
lastChildHash = firstChildHash;
}
} else if (insertAfter.nextSiblingHash == null) {
// last element
insertAfter.nextSiblingHash = newChild.storageKey;
insertAfter.invalidate();
lastChildHash = newChild.storageKey;
} else {
insertAfter.nextSiblingHash = newChild.storageKey;
insertAfter.invalidate();
}

newChild.parentHash = this.storageKey;
sd.put(newChild);
newChild.invalidate();
childrenCount++;
Expand Down Expand Up @@ -270,7 +305,7 @@ private PathElement[] decompactElement(PathElement pe) {
return new PathElement[]{parent, child};
}

private static byte[] getVirtualStorageKey(byte[] childStorageKey) {
public static byte[] getVirtualStorageKey(byte[] childStorageKey) {
BigInteger i = ByteUtil.bytesToBigInteger(childStorageKey).subtract(BigInteger.ONE);
return ByteUtil.bigIntegerToBytes(i, 32);
}
Expand Down Expand Up @@ -422,6 +457,14 @@ public byte[] serialize() {
RLP.encodeElement(encodeHash(lastChildHash))
);
}

PathElement copyLight() {
PathElement ret = new PathElement();
ret.type = type;
ret.key = key;
ret.storageKey = storageKey;
return ret;
}
// public byte[] serialize() {
// try {
// ObjectMapper om = new ObjectMapper();
Expand Down Expand Up @@ -464,13 +507,28 @@ public void store() {
dirtyNodes.clear();
}

KeyValueDataSource storageDb;
Map<ByteArrayWrapper, PathElement> cache = new HashMap<>();
public StorageDictionary getFiltered(Set<DataWord> hashFilter) {
HashMapDB filterSource = new HashMapDB();
StorageDictionary ret = new StorageDictionary(filterSource);
for (DataWord hash : hashFilter) {
PathElement pathElement = get(hash.getData());
ArrayList<PathElement> path = new ArrayList<>();
while(pathElement.type != Type.Root) {
path.add(0, pathElement.copyLight());
pathElement = pathElement.getParent();
}
ret.addPath(path.toArray(new PathElement[0]));
}
return ret;
}

private KeyValueDataSource storageDb;
private Map<ByteArrayWrapper, PathElement> cache = new HashMap<>();

List<PathElement> dirtyNodes = new ArrayList<>();
private List<PathElement> dirtyNodes = new ArrayList<>();

PathElement root;
boolean exist;
private PathElement root;
private boolean exist;

public StorageDictionary(KeyValueDataSource storageDb) {
this.storageDb = storageDb;
Expand Down Expand Up @@ -533,15 +591,6 @@ public PathElement deserializePathElement(byte[] bb) {
ret.lastChildHash= decodeHash(list.get(8).getRLPData());
return ret;
}
// public PathElement deserializePathElement(byte[] bb) {
// try {
// ObjectMapper om = new ObjectMapper();
// PathElement ret = om.readValue(new String(bb), PathElement.class);
// return ret;
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// }

public static byte[] toStorageKey(String hex) {
return Hex.decode(Utils.align(hex, '0', 64, false));
Expand Down
114 changes: 73 additions & 41 deletions ethereumj-core/src/main/java/org/ethereum/vm/StorageDictionaryHandler.java
View file
Expand Up @@ -8,6 +8,7 @@
import org.ethereum.db.StorageDictionaryDb;
import org.ethereum.util.Utils;
import org.ethereum.vm.program.Program;
import org.ethereum.vm.program.Stack;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spongycastle.util.encoders.Hex;
Expand All @@ -28,7 +29,7 @@
/**
* Created by Anton Nashatyrev on 04.09.2015.
*/
public class StorageDictionaryHandler implements VMHook {
public class StorageDictionaryHandler {
private static final Logger logger = LoggerFactory.getLogger("VM");

static class Entry {
Expand Down Expand Up @@ -114,23 +115,24 @@ public StorageDictionary.PathElement[] getKeyOriginSerpent(byte[] key) {
int pathLength = entry.input.length / 32;
StorageDictionary.PathElement[] ret = new StorageDictionary.PathElement[pathLength];
for (int i = 0; i < ret.length; i++) {
ret[i] = guessPathElement(Arrays.copyOfRange(entry.input, i * 32, (i+1) * 32))[0];
byte[] storageKey = SHA3Helper.sha3(entry.input, 0, (i + 1) * 32);
ret[i] = guessPathElement(Arrays.copyOfRange(entry.input, i * 32, (i+1) * 32), storageKey)[0];
ret[i].type = StorageDictionary.Type.MapKey;
}
return ret;
} else {
// not a Serenity contract
}
}
StorageDictionary.PathElement[] storageIndex = guessPathElement(key);
StorageDictionary.PathElement[] storageIndex = guessPathElement(key, key);
storageIndex[0].type = StorageDictionary.Type.StorageIndex;
return storageIndex;
}

public StorageDictionary.PathElement[] getKeyOriginSolidity(byte[] key) {
Entry entry = findHash(key);
if (entry == null) {
StorageDictionary.PathElement[] storageIndex = guessPathElement(key);
StorageDictionary.PathElement[] storageIndex = guessPathElement(key, key);
storageIndex[0].type = StorageDictionary.Type.StorageIndex;
storageIndex[0].storageKey = key;
return storageIndex;
Expand All @@ -139,22 +141,22 @@ public StorageDictionary.PathElement[] getKeyOriginSolidity(byte[] key) {
long offset = new BigInteger(key).subtract(new BigInteger(entry.hashValue.clone().getData())).longValue();
return Utils.mergeArrays(
getKeyOriginSolidity(Arrays.copyOfRange(entry.input, entry.input.length - 32, entry.input.length)),
guessPathElement(subKey), // hashKey = key - 1
guessPathElement(subKey, StorageDictionary.PathElement.getVirtualStorageKey(entry.hashValue.getData())), // hashKey = key - 1
new StorageDictionary.PathElement[] {new StorageDictionary.PathElement // hashKey = key
(subKey.length == 0 ? StorageDictionary.Type.ArrayIndex : StorageDictionary.Type.Offset, (int) offset, key)});
}
}

public StorageDictionary.PathElement[] guessPathElement(byte[] key) {
public StorageDictionary.PathElement[] guessPathElement(byte[] key, byte[] storageKey) {
if (key.length == 0) return new StorageDictionary.PathElement[0];
Object value = guessValue(key);
StorageDictionary.PathElement el = null;
if (value instanceof String) {
el = new StorageDictionary.PathElement((String) value, key);
el = new StorageDictionary.PathElement((String) value, storageKey);
} else if (value instanceof BigInteger) {
BigInteger bi = (BigInteger) value;
if (bi.bitLength() < 32) el = new StorageDictionary.PathElement(StorageDictionary.Type.MapKey, bi.intValue(), key);
else el = new StorageDictionary.PathElement("0x" + bi.toString(16), key);
if (bi.bitLength() < 32) el = new StorageDictionary.PathElement(StorageDictionary.Type.MapKey, bi.intValue(), storageKey);
else el = new StorageDictionary.PathElement("0x" + bi.toString(16), storageKey);
}
return new StorageDictionary.PathElement[] {el};
}
Expand Down Expand Up @@ -194,19 +196,26 @@ public static Object guessValue(byte[] bytes) {
static AtomicInteger cnt = new AtomicInteger();
static int lastExecuted;
public void dumpKeys(final ContractDetails storage) {
System.out.println("== StorageDictionaryHandler.dumpKeys: cur queue size: " + (cnt.get() - lastExecuted) + ", executed: " + lastExecuted);
// System.out.println("== StorageDictionaryHandler.dumpKeys: cur queue size: " + (cnt.get() - lastExecuted) + ", executed: " + lastExecuted);

executor.execute(new Runnable() {
int id = cnt.getAndIncrement();
@Override
public void run() {
// executor.execute(new Runnable() {
// int id = cnt.getAndIncrement();
//
// @Override
// public void run() {
StorageDictionary solidityDict = StorageDictionaryDb.INST.getOrCreate(StorageDictionaryDb.Layout.Solidity,
contractAddress);
StorageDictionary serpentDict = StorageDictionaryDb.INST.getOrCreate(StorageDictionaryDb.Layout.Serpent,
contractAddress);

String prevDump = null;
for (ByteArrayWrapper key : storeKeys.keySet()) {
solidityDict.addPath(getKeyOriginSolidity(key.getData()));
// try {
// prevDump = solidityDict.dump();
// } catch (Throwable e) {
// throw new RuntimeException(e);
// }
serpentDict.addPath(getKeyOriginSerpent(key.getData()));
}

Expand Down Expand Up @@ -260,9 +269,12 @@ public void run() {

// StorageDictionaryDb.INST.put(StorageDictionaryDb.Layout.Solidity, contractAddress, solidityDict);
// StorageDictionaryDb.INST.put(StorageDictionaryDb.Layout.Serpent, contractAddress, serpentDict);
lastExecuted = id;
}
});
solidityDict.store();
serpentDict.store();
StorageDictionaryDb.INST.flush();
// lastExecuted = id;
// }
// });
}

public void vmStartPlayNotify() {
Expand All @@ -277,32 +289,52 @@ public void vmEndPlayNotify(ContractDetails contractDetails) {
}
}

@Override
public void startPlay(Program program) {
vmStartPlayNotify();
}
public static VMHook HOOK = new VMHook() {
java.util.Stack<StorageDictionaryHandler> handlerStack = new java.util.Stack<>();
@Override
public void startPlay(Program program) {
try {
// System.out.println("Start play: " + program.getOwnerAddress());
handlerStack.push(new StorageDictionaryHandler(program.getOwnerAddress()));
// System.out.println("Started play: " + program.getOwnerAddress());
handlerStack.peek().vmStartPlayNotify();
} catch (Exception e) {
logger.error("Error within handler: ", e);
}
}

@Override
public void stopPlay(Program program) {
vmEndPlayNotify(program.getStorage().getContractDetails(program.getOwnerAddress().getLast20Bytes()));
}
@Override
public void stopPlay(Program program) {
try {
// System.out.println("Stop play: " + handler);
handlerStack.pop().vmEndPlayNotify(program.getStorage().getContractDetails(program.getOwnerAddress().getLast20Bytes()));
} catch (Exception e) {
logger.error("Error within handler: ", e);
}
}

public void step(Program program, OpCode opcode) {
switch (opcode) {
case SSTORE:
DataWord addr = program.getStack().get(0);
DataWord value = program.getStack().get(1);
vmSStoreNotify(addr, value);
break;
case SHA3:
DataWord memOffsetData = program.getStack().get(0);
DataWord lengthData = program.getStack().get(1);
byte[] buffer = program.memoryChunk(memOffsetData.intValue(), lengthData.intValue());
byte[] encoded = sha3(buffer);
DataWord word = new DataWord(encoded);
vmSha3Notify(buffer, word);
break;
public void step(Program program, OpCode opcode) {
try {
Stack stack = program.getStack();
switch (opcode) {
case SSTORE:
DataWord addr = stack.get(stack.size() - 1);
DataWord value = stack.get(stack.size() - 2);
handlerStack.peek().vmSStoreNotify(addr, value);
break;
case SHA3:
DataWord memOffsetData = stack.get(stack.size() - 1);
DataWord lengthData = stack.get(stack.size() - 2);
byte[] buffer = program.memoryChunk(memOffsetData.intValue(), lengthData.intValue());
byte[] encoded = sha3(buffer);
DataWord word = new DataWord(encoded);
handlerStack.peek().vmSha3Notify(buffer, word);
break;
}
} catch (Exception e) {
logger.error("Error within handler: ", e);
}
}
}
};

}

0 comments on commit 7d9d131

Please sign in to comment.