Permalink
Browse files

Switched state tree pruning to be based on reference counting

  • Loading branch information...
vbuterin committed Jun 27, 2017
1 parent 16e2a51 commit 8931e4945348ffa6e21cd77dba23846341056410
Showing with 80 additions and 45 deletions.
  1. +59 −31 ethereum/db.py
  2. +5 −1 ethereum/genesis_helpers.py
  3. +4 −2 ethereum/pow/chain.py
  4. +1 −1 ethereum/securetrie.py
  5. +3 −3 ethereum/state.py
  6. +8 −7 ethereum/trie.py
View
@@ -1,6 +1,12 @@
from ethereum import utils
from ethereum.slogging import get_logger
from rlp.utils import str_to_bytes
import sys
if sys.version_info.major == 2:
from repoze.lru import lru_cache
else:
from functools import lru_cache
log = get_logger('db')
@@ -41,25 +47,6 @@ def __eq__(self, other):
def __hash__(self):
return utils.big_endian_to_int(str_to_bytes(self.__repr__()))
def inc_refcount(self, key, value):
self.put(key, value)
def dec_refcount(self, key):
pass
def revert_refcount_changes(self, epoch):
pass
def commit_refcount_changes(self, epoch):
pass
def cleanup(self, epoch):
pass
def put_temporarily(self, key, value):
self.inc_refcount(key, value)
self.dec_refcount(key)
DB = EphemDB = _EphemDB
@@ -136,21 +123,62 @@ def __eq__(self, other):
def __hash__(self):
return utils.big_endian_to_int(str_to_bytes(self.__repr__()))
def inc_refcount(self, key, value):
self.put(key, value)
@lru_cache(128)
def add1(b):
v = utils.big_endian_to_int(b)
return utils.zpad(utils.encode_int(v + 1), 4)
def dec_refcount(self, key):
pass
@lru_cache(128)
def sub1(b):
v = utils.big_endian_to_int(b)
return utils.zpad(utils.encode_int(v - 1), 4)
def revert_refcount_changes(self, epoch):
pass
class RefcountDB(BaseDB):
def commit_refcount_changes(self, epoch):
pass
def __init__(self, db):
self.db = db
self.kv = None
def get(self, key):
return self.db.get(key)[4:]
def cleanup(self, epoch):
def get_refcount(self, key):
try:
return utils.big_endian_to_int(self.db.get(key)[:4])
except KeyError:
return 0
def put(self, key, value):
try:
existing = self.db.get(key)
assert existing[4:] == value
self.db.put(key, add1(existing[:4]) + value)
# print('putin', key, utils.big_endian_to_int(existing[:4]) + 1)
except KeyError:
self.db.put(key, b'\x00\x00\x00\x01' + value)
# print('putin', key, 1)
def delete(self, key):
existing = self.db.get(key)
if existing[:4] == b'\x00\x00\x00\x01':
# print('deletung')
self.db.delete(key)
else:
# print(repr(existing[:4]))
self.db.put(key, sub1(existing[:4]) + existing[4:])
def commit(self):
pass
def put_temporarily(self, key, value):
self.inc_refcount(key, value)
self.dec_refcount(key)
def _has_key(self, key):
return key in self.db
def __contains__(self, key):
return self._has_key(key)
def __eq__(self, other):
return isinstance(other, self.__class__) and self.db == other.db
def __hash__(self):
return utils.big_endian_to_int(str_to_bytes(self.__repr__()))
@@ -4,7 +4,7 @@
parse_as_bin, parse_as_int, normalize_address
from ethereum.config import Env
from ethereum.consensus_strategy import get_consensus_strategy
from ethereum.db import OverlayDB
from ethereum.db import OverlayDB, RefcountDB
import rlp
import json
@@ -44,6 +44,10 @@ def state_from_genesis_declaration(genesis_data, env, block=None, allow_empties=
state.set_storage_data(addr, big_endian_to_int(parse_as_bin(k)), big_endian_to_int(parse_as_bin(v)))
get_consensus_strategy(state.config).initialize(state, block)
state.commit(allow_empties=allow_empties)
print('deleting %d' % len(state.deletes))
rdb = RefcountDB(state.db)
for delete in state.deletes:
rdb.delete(delete)
block.header.state_root = state.trie.root_hash
state.prev_headers=[block.header]
return state
View
@@ -18,6 +18,7 @@
from ethereum.pow.consensus import initialize
from ethereum.genesis_helpers import mk_basic_state, state_from_genesis_declaration, \
initialize_genesis_keys
from ethereum.db import RefcountDB
log = get_logger('eth.chain')
@@ -330,9 +331,10 @@ def add_block(self, block):
if old_block_hash:
try:
deletes = self.db.get(b'deletes:'+old_block_hash)
print('Deleting %d trie nodes' % (len(deletes) // 32))
print('Deleting up to %d trie nodes' % (len(deletes) // 32))
rdb = RefcountDB(self.db)
for i in range(0, len(deletes), 32):
self.db.delete(deletes[i: i+32])
rdb.delete(deletes[i: i+32])
self.db.delete(b'deletes:'+old_block_hash)
except KeyError as e:
print(e)
View
@@ -9,7 +9,7 @@ def __init__(self, t):
def update(self, k, v):
h = utils.sha3(k)
self.db.put(h, k)
self.db.put(h, utils.str_to_bytes(k))
self.trie.update(h, v)
def get(self, k):
View
@@ -10,7 +10,7 @@
from ethereum.securetrie import SecureTrie
from ethereum.config import default_config, Env
from ethereum.block import FakeHeader
from ethereum.db import BaseDB, EphemDB, OverlayDB
from ethereum.db import BaseDB, EphemDB, OverlayDB, RefcountDB
from ethereum.specials import specials as default_specials
import copy
import sys
@@ -64,7 +64,7 @@ def __init__(self, nonce, balance, storage, code_hash, env, address):
self.address = address
super(Account, self).__init__(nonce, balance, storage, code_hash)
self.storage_cache = {}
self.storage_trie = SecureTrie(Trie(self.env.db, prefix=address))
self.storage_trie = SecureTrie(Trie(RefcountDB(self.env.db)))
self.storage_trie.root_hash = self.storage
self.touched = False
self.existent_at_start = True
@@ -131,7 +131,7 @@ class State():
def __init__(self, root=b'', env=Env(), **kwargs):
self.env = env
self.trie = SecureTrie(Trie(self.db, root))
self.trie = SecureTrie(Trie(RefcountDB(self.db), root))
for k, v in STATE_DEFAULTS.items():
setattr(self, k, kwargs.get(k, copy.copy(v)))
self.journal = []
View
@@ -139,7 +139,7 @@ def is_key_value_type(node_type):
class Trie(object):
def __init__(self, db, root_hash=BLANK_ROOT, prefix=b''):
def __init__(self, db, root_hash=BLANK_ROOT):
"""it also present a dictionary like interface
:param db key value database
@@ -148,7 +148,6 @@ def __init__(self, db, root_hash=BLANK_ROOT, prefix=b''):
self.db = db # Pass in a database object directly
self.set_root_hash(root_hash)
self.deletes = []
self.prefix = prefix
# def __init__(self, dbfile, root_hash=BLANK_ROOT):
# """it also present a dictionary like interface
@@ -175,7 +174,7 @@ def get_root_hash(self):
def _update_root_hash(self):
val = rlp_encode(self.root_node)
key = utils.sha3(val)
self.db.put(self.prefix+key, val)
self.db.put(key, str_to_bytes(val))
self._root_hash = key
@root_hash.setter
@@ -208,7 +207,7 @@ def _delete_child_storage(self, node):
elif node_type == NODE_TYPE_EXTENSION:
self._delete_child_storage(self._decode_to_node(node[1]))
def _encode_node(self, node):
def _encode_node(self, node, put_in_db=True):
if node == BLANK_NODE:
return BLANK_NODE
# assert isinstance(node, list)
@@ -217,15 +216,16 @@ def _encode_node(self, node):
return node
hashkey = utils.sha3(rlpnode)
self.db.put(self.prefix+hashkey, rlpnode)
if put_in_db:
self.db.put(hashkey, str_to_bytes(rlpnode))
return hashkey
def _decode_to_node(self, encoded):
if encoded == BLANK_NODE:
return BLANK_NODE
if isinstance(encoded, list):
return encoded
o = rlp.decode(self.db.get(self.prefix+encoded))
o = rlp.decode(self.db.get(encoded))
return o
def _get_node_type(self, node):
@@ -600,7 +600,7 @@ def _delete_node_storage(self, node):
if node == BLANK_NODE:
return
# assert isinstance(node, list)
encoded = self._encode_node(node)
encoded = self._encode_node(node, put_in_db=False)
if len(encoded) < 32:
return
"""
@@ -609,6 +609,7 @@ def _delete_node_storage(self, node):
thus we can not safely delete nodes for now
"""
self.deletes.append(encoded)
# print('del', encoded, self.db.get_refcount(encoded))
def _delete(self, node, key):
""" update item inside a node

0 comments on commit 8931e49

Please sign in to comment.