From 53a6b1c6892b13cbda56c6b8fad001df47e2ebb0 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 21 Nov 2018 18:44:57 -0600 Subject: [PATCH 01/11] port statediff from https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go; minor fixes --- statediff/helpers.go | 114 +++++++++++++ statediff/statediff.go | 67 ++++++++ statediff/statediff_builder.go | 301 +++++++++++++++++++++++++++++++++ 3 files changed, 482 insertions(+) create mode 100644 statediff/helpers.go create mode 100644 statediff/statediff.go create mode 100644 statediff/statediff_builder.go diff --git a/statediff/helpers.go b/statediff/helpers.go new file mode 100644 index 000000000000..976d21fe2896 --- /dev/null +++ b/statediff/helpers.go @@ -0,0 +1,114 @@ +package statediff + +import ( + "sort" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/trie" +) + +func sortKeys(data map[common.Address]*state.Account) []string { + var keys []string + for key := range data { + keys = append(keys, key.Hex()) + } + sort.Strings(keys) + + return keys +} + +func findIntersection(a, b []string) []string { + lenA := len(a) + lenB := len(b) + iOfA, iOfB := 0, 0 + updates := make([]string, 0) + if iOfA >= lenA || iOfB >= lenB { + return updates + } + for { + switch strings.Compare(a[iOfA], b[iOfB]) { + // a[iOfA] < b[iOfB] + case -1: + iOfA++ + if iOfA >= lenA { + return updates + } + break + // a[iOfA] == b[iOfB] + case 0: + updates = append(updates, a[iOfA]) + iOfA++ + iOfB++ + if iOfA >= lenA || iOfB >= lenB { + return updates + } + break + // a[iOfA] > b[iOfB] + case 1: + iOfB++ + if iOfB >= lenB { + return updates + } + break + } + } + +} + +func pathToStr(it trie.NodeIterator) string { + path := it.Path() + if hasTerm(path) { + path = path[:len(path)-1] + } + nibblePath := "" + for i, v := range common.ToHex(path) { + if i%2 == 0 && i > 1 { + continue + } + nibblePath = nibblePath + string(v) + } + + return nibblePath +} + +// Duplicated from trie/encoding.go +func hexToKeybytes(hex []byte) []byte { + if hasTerm(hex) { + hex = hex[:len(hex)-1] + } + if len(hex)&1 != 0 { + panic("can't convert hex key of odd length") + } + key := make([]byte, (len(hex)+1)/2) + decodeNibbles(hex, key) + + return key +} + +func decodeNibbles(nibbles []byte, bytes []byte) { + for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { + bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] + } +} + +// prefixLen returns the length of the common prefix of a and b. +func prefixLen(a, b []byte) int { + var i, length = 0, len(a) + if len(b) < length { + length = len(b) + } + for ; i < length; i++ { + if a[i] != b[i] { + break + } + } + + return i +} + +// hasTerm returns whether a hex key has the terminator flag. +func hasTerm(s []byte) bool { + return len(s) > 0 && s[len(s)-1] == 16 +} diff --git a/statediff/statediff.go b/statediff/statediff.go new file mode 100644 index 000000000000..d980ef867d34 --- /dev/null +++ b/statediff/statediff.go @@ -0,0 +1,67 @@ +package statediff + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +type StateDiff struct { + BlockNumber int64 `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts map[common.Address]AccountDiffEventual `json:"createdAccounts" gencodec:"required"` + DeletedAccounts map[common.Address]AccountDiffEventual `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts map[common.Address]AccountDiffIncremental `json:"updatedAccounts" gencodec:"required"` + + encoded []byte + err error +} + +func (self *StateDiff) ensureEncoded() { + if self.encoded == nil && self.err == nil { + self.encoded, self.err = json.Marshal(self) + } +} + +// Implement Encoder interface for StateDiff +func (sd *StateDiff) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Implement Encoder interface for StateDiff +func (sd *StateDiff) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err +} + +type AccountDiffEventual struct { + Nonce diffUint64 `json:"nonce" gencodec:"required"` + Balance diffBigInt `json:"balance" gencodec:"required"` + Code string `json:"code" gencodec:"required"` + CodeHash string `json:"codeHash" gencodec:"required"` + ContractRoot diffString `json:"contractRoot" gencodec:"required"` + Storage map[string]diffString `json:"storage" gencodec:"required"` +} + +type AccountDiffIncremental struct { + Nonce diffUint64 `json:"nonce" gencodec:"required"` + Balance diffBigInt `json:"balance" gencodec:"required"` + CodeHash string `json:"codeHash" gencodec:"required"` + ContractRoot diffString `json:"contractRoot" gencodec:"required"` + Storage map[string]diffString `json:"storage" gencodec:"required"` +} + +type diffString struct { + NewValue *string `json:"newValue" gencodec:"optional"` + OldValue *string `json:"oldValue" gencodec:"optional"` +} +type diffUint64 struct { + NewValue *uint64 `json:"newValue" gencodec:"optional"` + OldValue *uint64 `json:"oldValue" gencodec:"optional"` +} +type diffBigInt struct { + NewValue *big.Int `json:"newValue" gencodec:"optional"` + OldValue *big.Int `json:"oldValue" gencodec:"optional"` +} diff --git a/statediff/statediff_builder.go b/statediff/statediff_builder.go new file mode 100644 index 000000000000..c3011ed66239 --- /dev/null +++ b/statediff/statediff_builder.go @@ -0,0 +1,301 @@ +package statediff + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +type StateDiffBuilder interface { + CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) +} + +type stateDiffBuilder struct { + chainDB ethdb.Database + trieDB *trie.Database + cachedTrie *trie.Trie +} + +func NewStateDiffBuilder(db ethdb.Database) *stateDiffBuilder { + return &stateDiffBuilder{ + chainDB: db, + trieDB: trie.NewDatabase(db), + } +} + +func (sdb *stateDiffBuilder) CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { + // Generate tries for old and new states + oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) + if err != nil { + return nil, err + } + newTrie, err := trie.New(newStateRoot, sdb.trieDB) + if err != nil { + return nil, err + } + + // Find created accounts + oldIt := oldTrie.NodeIterator([]byte{}) + newIt := newTrie.NodeIterator([]byte{}) + creations, err := sdb.collectDiffNodes(oldIt, newIt) + if err != nil { + return nil, err + } + + // Find deleted accounts + oldIt = oldTrie.NodeIterator(make([]byte, 0)) + newIt = newTrie.NodeIterator(make([]byte, 0)) + deletions, err := sdb.collectDiffNodes(newIt, oldIt) + if err != nil { + return nil, err + } + + // Find all the diffed keys + createKeys := sortKeys(creations) + deleteKeys := sortKeys(deletions) + updatedKeys := findIntersection(createKeys, deleteKeys) + + // Build and return the statediff + updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, &updatedKeys) + if err != nil { + return nil, err + } + createdAccounts, err := sdb.buildDiffEventual(creations, true) + if err != nil { + return nil, err + } + deletedAccounts, err := sdb.buildDiffEventual(deletions, false) + if err != nil { + return nil, err + } + + return &StateDiff{ + BlockNumber: blockNumber, + BlockHash: blockHash, + CreatedAccounts: createdAccounts, + DeletedAccounts: deletedAccounts, + UpdatedAccounts: updatedAccounts, + }, nil +} + +func (sdb *stateDiffBuilder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { + var diffAccounts map[common.Address]*state.Account + it, _ := trie.NewDifferenceIterator(a, b) + + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", common.Hash(it.Hash())) + if it.Leaf() { + + // lookup address + path := make([]byte, len(it.Path())-1) + copy(path, it.Path()) + addr, err := sdb.addressByPath(path) + if err != nil { + log.Error("Error looking up address via path", "path", path, "error", err) + return nil, err + } + + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil { + log.Error("Error looking up account via address", "address", addr, "error", err) + return nil, err + } + + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", addr, "account", account) + diffAccounts[*addr] = &account + } + cont := it.Next(true) + if !cont { + break + } + } + return diffAccounts, nil +} + +func (sdb *stateDiffBuilder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) { + accountDiffs := make(map[common.Address]AccountDiffEventual) + for addr, val := range accounts { + sr := val.Root + if storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created); err != nil { + log.Error("Failed building eventual storage diffs", "Address", val, "error", err) + return nil, err + } else { + code := "" + codeBytes, err := sdb.chainDB.Get(val.CodeHash) + if err == nil && len(codeBytes) != 0 { + code = common.ToHex(codeBytes) + } else { + log.Debug("No code field.", "codehash", val.CodeHash, "Address", val, "error", err) + } + codeHash := common.ToHex(val.CodeHash) + if created { + nonce := diffUint64{ + NewValue: &val.Nonce, + } + + balance := diffBigInt{ + NewValue: val.Balance, + } + + hexRoot := val.Root.Hex() + contractRoot := diffString{ + NewValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + Code: code, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + } else { + nonce := diffUint64{ + OldValue: &val.Nonce, + } + balance := diffBigInt{ + OldValue: val.Balance, + } + hexRoot := val.Root.Hex() + contractRoot := diffString{ + OldValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + } + } + } + return accountDiffs, nil +} + +func (sdb *stateDiffBuilder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) { + updatedAccounts := make(map[common.Address]AccountDiffIncremental) + for _, val := range *updatedKeys { + createdAcc := creations[common.HexToAddress(val)] + deletedAcc := deletions[common.HexToAddress(val)] + oldSR := deletedAcc.Root + newSR := createdAcc.Root + if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil { + log.Error("Failed building storage diffs", "Address", val, "error", err) + return nil, err + } else { + nonce := diffUint64{ + NewValue: &createdAcc.Nonce, + OldValue: &deletedAcc.Nonce, + } + + balance := diffBigInt{ + NewValue: createdAcc.Balance, + OldValue: deletedAcc.Balance, + } + codeHash := common.ToHex(createdAcc.CodeHash) + + nHexRoot := createdAcc.Root.Hex() + oHexRoot := deletedAcc.Root.Hex() + contractRoot := diffString{ + NewValue: &nHexRoot, + OldValue: &oHexRoot, + } + + updatedAccounts[common.HexToAddress(val)] = AccountDiffIncremental{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + delete(creations, common.HexToAddress(val)) + delete(deletions, common.HexToAddress(val)) + } + } + return updatedAccounts, nil +} + +func (sdb *stateDiffBuilder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) { + log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) + sTrie, err := trie.New(sr, sdb.trieDB) + if err != nil { + return nil, err + } + it := sTrie.NodeIterator(make([]byte, 0)) + storageDiffs := make(map[string]diffString) + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + path := pathToStr(it) + value := common.ToHex(it.LeafBlob()) + if creation { + storageDiffs[path] = diffString{NewValue: &value} + } else { + storageDiffs[path] = diffString{OldValue: &value} + } + } + cont := it.Next(true) + if !cont { + break + } + } + return storageDiffs, nil +} + +func (sdb *stateDiffBuilder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) { + log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) + oldTrie, err := trie.New(oldSR, sdb.trieDB) + if err != nil { + return nil, err + } + newTrie, err := trie.New(newSR, sdb.trieDB) + if err != nil { + return nil, err + } + + oldIt := oldTrie.NodeIterator(make([]byte, 0)) + newIt := newTrie.NodeIterator(make([]byte, 0)) + it, _ := trie.NewDifferenceIterator(oldIt, newIt) + storageDiffs := make(map[string]diffString) + for { + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + path := pathToStr(it) + value := common.ToHex(it.LeafBlob()) + if oldVal, err := oldTrie.TryGet(it.LeafKey()); err != nil { + log.Error("Failed to look up value in oldTrie", "path", path, "error", err) + } else { + hexOldVal := common.ToHex(oldVal) + storageDiffs[path] = diffString{OldValue: &hexOldVal, NewValue: &value} + } + } + + cont := it.Next(true) + if !cont { + break + } + } + return storageDiffs, nil +} + +func (sdb *stateDiffBuilder) addressByPath(path []byte) (*common.Address, error) { + // db := core.PreimageTable(sdb.chainDb) + log.Debug("Looking up address from path", "path", common.ToHex(append([]byte("secure-key-"), path...))) + // if addrBytes,err := db.Get(path); err != nil { + if addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeybytes(path)...)); err != nil { + log.Error("Error looking up address via path", "path", common.ToHex(append([]byte("secure-key-"), path...)), "error", err) + return nil, err + } else { + addr := common.BytesToAddress(addrBytes) + log.Debug("Address found", "Address", addr) + return &addr, nil + } + +} From 26e8088cc5f676bcdbd51c889a8303d4603cefd8 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sun, 25 Nov 2018 23:52:53 -0600 Subject: [PATCH 02/11] integrating state diff extracting, building, and persisting into geth processes --- cmd/geth/main.go | 1 + cmd/utils/flags.go | 17 ++++ core/blockchain.go | 14 ++++ eth/backend.go | 8 +- eth/config.go | 4 + .../{statediff_builder.go => builder.go} | 43 +++++++--- statediff/builder_test.go | 21 +++++ statediff/config.go | 80 +++++++++++++++++++ statediff/extractor.go | 50 ++++++++++++ statediff/extractor_test.go | 20 +++++ statediff/helpers.go | 19 +++++ statediff/persister.go | 40 ++++++++++ statediff/persister_test.go | 20 +++++ statediff/statediff.go | 19 +++++ 14 files changed, 343 insertions(+), 13 deletions(-) rename statediff/{statediff_builder.go => builder.go} (78%) create mode 100644 statediff/builder_test.go create mode 100644 statediff/config.go create mode 100644 statediff/extractor.go create mode 100644 statediff/extractor_test.go create mode 100644 statediff/persister.go create mode 100644 statediff/persister_test.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 69802a48a763..15f76554851c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -132,6 +132,7 @@ var ( utils.GpoPercentileFlag, utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, + utils.StateDiffFlag, configFileFlag, } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d7b698c7e2e5..c18d98383fcb 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -626,6 +626,12 @@ var ( Usage: "External EVM configuration (default = built-in interpreter)", Value: "", } + + // Statediff flags + StateDiffFlag = cli.BoolFlag{ + Name: "statediff", + Usage: "Enables the calculation of state diffs between each block, persists these state diffs in ipfs", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1127,6 +1133,13 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { } } +// Check if state diff flags are on and applies them to eth context +func setStateDiff(ctx *cli.Context, cfg *eth.Config) { + if ctx.GlobalBool(StateDiffFlag.Name) && cfg.NoPruning && cfg.SyncMode == downloader.FullSync { + cfg.StateDiff.On = true + } +} + // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags @@ -1162,6 +1175,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" + if ctx.GlobalIsSet(StateDiffFlag.Name) { + setStateDiff(ctx, cfg) + } + if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } diff --git a/core/blockchain.go b/core/blockchain.go index d173b2de294d..fed6b4b74dae 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -20,6 +20,7 @@ package core import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/statediff" "io" "math/big" mrand "math/rand" @@ -72,6 +73,7 @@ type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + StateDiff bool // Whether or not to calculate and persist state diffs } // BlockChain represents the canonical chain given a database with a genesis @@ -133,6 +135,8 @@ type BlockChain struct { badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. + + diffExtractor statediff.Extractor // State diff processing interface } // NewBlockChain returns a fully initialised block chain using information @@ -173,6 +177,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) + if cacheConfig.StateDiff { + bc.diffExtractor = statediff.NewExtractor(db) + } + var err error bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { @@ -1187,6 +1195,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty parent = chain[i-1] } state, err := state.New(parent.Root(), bc.stateCache) + if err != nil { return i, events, coalescedLogs, err } @@ -1204,6 +1213,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } proctime := time.Since(bstart) + // If extracting statediffs, do so now + if bc.cacheConfig.StateDiff { + bc.diffExtractor.Extract(*parent, *block) + } + // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) if err != nil { diff --git a/eth/backend.go b/eth/backend.go index 4721408425cb..d9617564351f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -154,7 +154,13 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { EWASMInterpreter: config.EWASMInterpreter, EVMInterpreter: config.EVMInterpreter, } - cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout} + cacheConfig = &core.CacheConfig{ + Disabled: config.NoPruning, + TrieCleanLimit: config.TrieCleanCache, + TrieDirtyLimit: config.TrieDirtyCache, + TrieTimeLimit: config.TrieTimeout, + StateDiff: config.StateDiff.On, + } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) if err != nil { diff --git a/eth/config.go b/eth/config.go index 601f4735e33f..c675b6d08cf6 100644 --- a/eth/config.go +++ b/eth/config.go @@ -17,6 +17,7 @@ package eth import ( + "github.com/ethereum/go-ethereum/statediff" "math/big" "os" "os/user" @@ -128,6 +129,9 @@ type Config struct { EWASMInterpreter string // Type of the EVM interpreter ("" for default) EVMInterpreter string + + // Config for state diff building + StateDiff statediff.Config } type configMarshaling struct { diff --git a/statediff/statediff_builder.go b/statediff/builder.go similarity index 78% rename from statediff/statediff_builder.go rename to statediff/builder.go index c3011ed66239..9017c240dd67 100644 --- a/statediff/statediff_builder.go +++ b/statediff/builder.go @@ -1,3 +1,22 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + package statediff import ( @@ -9,24 +28,24 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -type StateDiffBuilder interface { - CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) +type Builder interface { + BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) } -type stateDiffBuilder struct { +type builder struct { chainDB ethdb.Database trieDB *trie.Database cachedTrie *trie.Trie } -func NewStateDiffBuilder(db ethdb.Database) *stateDiffBuilder { - return &stateDiffBuilder{ +func NewBuilder(db ethdb.Database) *builder { + return &builder{ chainDB: db, trieDB: trie.NewDatabase(db), } } -func (sdb *stateDiffBuilder) CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { +func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { // Generate tries for old and new states oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) if err != nil { @@ -81,7 +100,7 @@ func (sdb *stateDiffBuilder) CreateStateDiff(oldStateRoot, newStateRoot common.H }, nil } -func (sdb *stateDiffBuilder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { +func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { var diffAccounts map[common.Address]*state.Account it, _ := trie.NewDifferenceIterator(a, b) @@ -117,7 +136,7 @@ func (sdb *stateDiffBuilder) collectDiffNodes(a, b trie.NodeIterator) (map[commo return diffAccounts, nil } -func (sdb *stateDiffBuilder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) { +func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) { accountDiffs := make(map[common.Address]AccountDiffEventual) for addr, val := range accounts { sr := val.Root @@ -178,7 +197,7 @@ func (sdb *stateDiffBuilder) buildDiffEventual(accounts map[common.Address]*stat return accountDiffs, nil } -func (sdb *stateDiffBuilder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) { +func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) { updatedAccounts := make(map[common.Address]AccountDiffIncremental) for _, val := range *updatedKeys { createdAcc := creations[common.HexToAddress(val)] @@ -221,7 +240,7 @@ func (sdb *stateDiffBuilder) buildDiffIncremental(creations map[common.Address]* return updatedAccounts, nil } -func (sdb *stateDiffBuilder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) { +func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) { log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) sTrie, err := trie.New(sr, sdb.trieDB) if err != nil { @@ -249,7 +268,7 @@ func (sdb *stateDiffBuilder) buildStorageDiffsEventual(sr common.Hash, creation return storageDiffs, nil } -func (sdb *stateDiffBuilder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) { +func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) { log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) oldTrie, err := trie.New(oldSR, sdb.trieDB) if err != nil { @@ -285,7 +304,7 @@ func (sdb *stateDiffBuilder) buildStorageDiffsIncremental(oldSR common.Hash, new return storageDiffs, nil } -func (sdb *stateDiffBuilder) addressByPath(path []byte) (*common.Address, error) { +func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { // db := core.PreimageTable(sdb.chainDb) log.Debug("Looking up address from path", "path", common.ToHex(append([]byte("secure-key-"), path...))) // if addrBytes,err := db.Get(path); err != nil { diff --git a/statediff/builder_test.go b/statediff/builder_test.go new file mode 100644 index 000000000000..6433b167a44d --- /dev/null +++ b/statediff/builder_test.go @@ -0,0 +1,21 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff_test + diff --git a/statediff/config.go b/statediff/config.go new file mode 100644 index 000000000000..ee31f271ecb9 --- /dev/null +++ b/statediff/config.go @@ -0,0 +1,80 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import "fmt" + +type Config struct { + On bool + Mode StateDiffMode +} + +type StateDiffMode int + +const ( + IPFS StateDiffMode = iota + LDB + SQL +) + +func (mode StateDiffMode) IsValid() bool { + return mode >= IPFS && mode <= SQL +} + +// String implements the stringer interface. +func (mode StateDiffMode) String() string { + switch mode { + case IPFS: + return "ipfs" + case LDB: + return "ldb" + case SQL: + return "sql" + default: + return "unknown" + } +} + +func (mode StateDiffMode) MarshalText() ([]byte, error) { + switch mode { + case IPFS: + return []byte("ipfs"), nil + case LDB: + return []byte("ldb"), nil + case SQL: + return []byte("sql"), nil + default: + return nil, fmt.Errorf("unknown state diff storage mode %d", mode) + } +} + +func (mode *StateDiffMode) UnmarshalText(text []byte) error { + switch string(text) { + case "ipfs": + *mode = IPFS + case "ldb": + *mode = LDB + case "sql": + *mode = SQL + default: + return fmt.Errorf(`unknown state diff storage mode %q, want "ipfs", "ldb" or "sql"`, text) + } + return nil +} \ No newline at end of file diff --git a/statediff/extractor.go b/statediff/extractor.go new file mode 100644 index 000000000000..2abe3ebb3b49 --- /dev/null +++ b/statediff/extractor.go @@ -0,0 +1,50 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" +) + +type Extractor interface { + ExtractStateDiff(parent, current types.Block) error +} + +type extractor struct { + b *builder + p *persister +} + +func NewExtractor(db ethdb.Database) *extractor { + return &extractor{ + b: NewBuilder(db), + p: NewPersister(), + } +} + +func (e *extractor) Extract(parent, current types.Block) error { + stateDiff, err := e.b.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) + if err != nil { + return err + } + + return e.p.PersistStateDiff(stateDiff) +} \ No newline at end of file diff --git a/statediff/extractor_test.go b/statediff/extractor_test.go new file mode 100644 index 000000000000..de5e170957b0 --- /dev/null +++ b/statediff/extractor_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff_test \ No newline at end of file diff --git a/statediff/helpers.go b/statediff/helpers.go index 976d21fe2896..50626ab2c4ca 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -1,3 +1,22 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + package statediff import ( diff --git a/statediff/persister.go b/statediff/persister.go new file mode 100644 index 000000000000..a3eba08260f9 --- /dev/null +++ b/statediff/persister.go @@ -0,0 +1,40 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +type Persister interface { + PersistStateDiff(sd *StateDiff) error +} + +type persister struct { + +} + +func NewPersister() *persister { + return &persister{ + + } +} + +func (p *persister) PersistStateDiff(sd *StateDiff) error { + //TODO: Persist state diff in IPFS + + return nil +} \ No newline at end of file diff --git a/statediff/persister_test.go b/statediff/persister_test.go new file mode 100644 index 000000000000..de5e170957b0 --- /dev/null +++ b/statediff/persister_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff_test \ No newline at end of file diff --git a/statediff/statediff.go b/statediff/statediff.go index d980ef867d34..f2e606936bbd 100644 --- a/statediff/statediff.go +++ b/statediff/statediff.go @@ -1,3 +1,22 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + package statediff import ( From 976bff614a201ff33af0bc0286fb568c20b4c725 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 26 Nov 2018 02:37:25 -0600 Subject: [PATCH 03/11] work towards persisting created statediffs in ipfs; based off github.com/vulcanize/eth-block-extractor --- core/blockchain.go | 21 ++-- statediff/builder.go | 98 +++++++++---------- statediff/config.go | 15 +-- statediff/extractor.go | 27 ++--- statediff/helpers.go | 17 +--- statediff/ipfs/adder.go | 58 +++++++++++ statediff/ipfs/dag_putter.go | 79 +++++++++++++++ statediff/{persister.go => ipfs/helpers.go} | 36 ++++--- statediff/ipfs/node.go | 78 +++++++++++++++ statediff/publisher.go | 63 ++++++++++++ .../{persister_test.go => publisher_test.go} | 0 statediff/{statediff.go => struct.go} | 2 +- 12 files changed, 383 insertions(+), 111 deletions(-) create mode 100644 statediff/ipfs/adder.go create mode 100644 statediff/ipfs/dag_putter.go rename statediff/{persister.go => ipfs/helpers.go} (71%) create mode 100644 statediff/ipfs/node.go create mode 100644 statediff/publisher.go rename statediff/{persister_test.go => publisher_test.go} (100%) rename statediff/{statediff.go => struct.go} (98%) diff --git a/core/blockchain.go b/core/blockchain.go index fed6b4b74dae..77295b888cf2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -20,7 +20,6 @@ package core import ( "errors" "fmt" - "github.com/ethereum/go-ethereum/statediff" "io" "math/big" mrand "math/rand" @@ -45,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" + "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -73,7 +73,7 @@ type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - StateDiff bool // Whether or not to calculate and persist state diffs + StateDiff statediff.Config // Settings for state diff extraction } // BlockChain represents the canonical chain given a database with a genesis @@ -177,11 +177,14 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) - if cacheConfig.StateDiff { - bc.diffExtractor = statediff.NewExtractor(db) + var err error + if cacheConfig.StateDiff.On { + bc.diffExtractor, err = statediff.NewExtractor(db, cacheConfig.StateDiff) + if err != nil { + return nil, err + } } - var err error bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { return nil, err @@ -1214,8 +1217,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty proctime := time.Since(bstart) // If extracting statediffs, do so now - if bc.cacheConfig.StateDiff { - bc.diffExtractor.Extract(*parent, *block) + if bc.cacheConfig.StateDiff.On { + // Currently not doing anything with returned cid... + _, err = bc.diffExtractor.ExtractStateDiff(*parent, *block) + if err != nil { + return i, events, coalescedLogs, err + } } // Write the block to the chain and get the status. diff --git a/statediff/builder.go b/statediff/builder.go index 9017c240dd67..433353591f5f 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -78,7 +78,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block updatedKeys := findIntersection(createKeys, deleteKeys) // Build and return the statediff - updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, &updatedKeys) + updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { return nil, err } @@ -133,6 +133,7 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address break } } + return diffAccounts, nil } @@ -140,66 +141,63 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account accountDiffs := make(map[common.Address]AccountDiffEventual) for addr, val := range accounts { sr := val.Root - if storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created); err != nil { + storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created) + if err != nil { log.Error("Failed building eventual storage diffs", "Address", val, "error", err) return nil, err + } + + codeBytes, err := sdb.chainDB.Get(val.CodeHash) + + codeHash := common.ToHex(val.CodeHash) + hexRoot := val.Root.Hex() + + if created { + nonce := diffUint64{ + NewValue: &val.Nonce, + } + + balance := diffBigInt{ + NewValue: val.Balance, + } + + contractRoot := diffString{ + NewValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + Code: codeBytes, + ContractRoot: contractRoot, + Storage: storageDiffs, + } } else { - code := "" - codeBytes, err := sdb.chainDB.Get(val.CodeHash) - if err == nil && len(codeBytes) != 0 { - code = common.ToHex(codeBytes) - } else { - log.Debug("No code field.", "codehash", val.CodeHash, "Address", val, "error", err) + nonce := diffUint64{ + OldValue: &val.Nonce, } - codeHash := common.ToHex(val.CodeHash) - if created { - nonce := diffUint64{ - NewValue: &val.Nonce, - } - - balance := diffBigInt{ - NewValue: val.Balance, - } - - hexRoot := val.Root.Hex() - contractRoot := diffString{ - NewValue: &hexRoot, - } - accountDiffs[addr] = AccountDiffEventual{ - Nonce: nonce, - Balance: balance, - CodeHash: codeHash, - Code: code, - ContractRoot: contractRoot, - Storage: storageDiffs, - } - } else { - nonce := diffUint64{ - OldValue: &val.Nonce, - } - balance := diffBigInt{ - OldValue: val.Balance, - } - hexRoot := val.Root.Hex() - contractRoot := diffString{ - OldValue: &hexRoot, - } - accountDiffs[addr] = AccountDiffEventual{ - Nonce: nonce, - Balance: balance, - CodeHash: codeHash, - ContractRoot: contractRoot, - Storage: storageDiffs, - } + balance := diffBigInt{ + OldValue: val.Balance, + } + contractRoot := diffString{ + OldValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, } } } + return accountDiffs, nil } -func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) { +func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys []string) (map[common.Address]AccountDiffIncremental, error) { updatedAccounts := make(map[common.Address]AccountDiffIncremental) - for _, val := range *updatedKeys { + for _, val := range updatedKeys { createdAcc := creations[common.HexToAddress(val)] deletedAcc := deletions[common.HexToAddress(val)] oldSR := deletedAcc.Root diff --git a/statediff/config.go b/statediff/config.go index ee31f271ecb9..ac207acd1347 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -22,26 +22,27 @@ package statediff import "fmt" type Config struct { - On bool - Mode StateDiffMode + On bool // Whether or not to extract state diffs + Mode StateDiffMode // Mode for storing diffs + Path string // Path for storing diffs } type StateDiffMode int const ( - IPFS StateDiffMode = iota + IPLD StateDiffMode = iota LDB SQL ) func (mode StateDiffMode) IsValid() bool { - return mode >= IPFS && mode <= SQL + return mode >= IPLD && mode <= SQL } // String implements the stringer interface. func (mode StateDiffMode) String() string { switch mode { - case IPFS: + case IPLD: return "ipfs" case LDB: return "ldb" @@ -54,7 +55,7 @@ func (mode StateDiffMode) String() string { func (mode StateDiffMode) MarshalText() ([]byte, error) { switch mode { - case IPFS: + case IPLD: return []byte("ipfs"), nil case LDB: return []byte("ldb"), nil @@ -68,7 +69,7 @@ func (mode StateDiffMode) MarshalText() ([]byte, error) { func (mode *StateDiffMode) UnmarshalText(text []byte) error { switch string(text) { case "ipfs": - *mode = IPFS + *mode = IPLD case "ldb": *mode = LDB case "sql": diff --git a/statediff/extractor.go b/statediff/extractor.go index 2abe3ebb3b49..3cbb5a311604 100644 --- a/statediff/extractor.go +++ b/statediff/extractor.go @@ -25,26 +25,31 @@ import ( ) type Extractor interface { - ExtractStateDiff(parent, current types.Block) error + ExtractStateDiff(parent, current types.Block) (string, error) } type extractor struct { - b *builder - p *persister + *builder // Interface for building state diff objects from two blocks + *publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) } -func NewExtractor(db ethdb.Database) *extractor { - return &extractor{ - b: NewBuilder(db), - p: NewPersister(), +func NewExtractor(db ethdb.Database, config Config) (*extractor, error) { + publisher, err := NewPublisher(config) + if err != nil { + return nil, err } + + return &extractor{ + builder: NewBuilder(db), + publisher: publisher, + }, nil } -func (e *extractor) Extract(parent, current types.Block) error { - stateDiff, err := e.b.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) +func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) { + stateDiff, err := e.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) if err != nil { - return err + return "", err } - return e.p.PersistStateDiff(stateDiff) + return e.PublishStateDiff(stateDiff) } \ No newline at end of file diff --git a/statediff/helpers.go b/statediff/helpers.go index 50626ab2c4ca..8a0a6466e172 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -112,22 +112,7 @@ func decodeNibbles(nibbles []byte, bytes []byte) { } } -// prefixLen returns the length of the common prefix of a and b. -func prefixLen(a, b []byte) int { - var i, length = 0, len(a) - if len(b) < length { - length = len(b) - } - for ; i < length; i++ { - if a[i] != b[i] { - break - } - } - - return i -} - // hasTerm returns whether a hex key has the terminator flag. func hasTerm(s []byte) bool { return len(s) > 0 && s[len(s)-1] == 16 -} +} \ No newline at end of file diff --git a/statediff/ipfs/adder.go b/statediff/ipfs/adder.go new file mode 100644 index 000000000000..23eea1c1ff54 --- /dev/null +++ b/statediff/ipfs/adder.go @@ -0,0 +1,58 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "context" + + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/repo/fsrepo" + ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" +) + +type Adder interface { + Add(node ipld.Node) error +} + +type adder struct { + n *core.IpfsNode + ctx context.Context +} + +func (a adder) Add(node ipld.Node) error { + return a.n.DAG.Add(a.n.Context(), node) // For some reason DAG.Add method is not being exposed by the ipld.DAGService +} + +func NewAdder(repoPath string) (*adder, error) { + r, err := fsrepo.Open(repoPath) + if err != nil { + return nil, err + } + ctx := context.Background() + cfg := &core.BuildCfg{ + Online: false, + Repo: r, + } + ipfsNode, err := core.NewNode(ctx, cfg) + if err != nil { + return nil, err + } + return &adder{n: ipfsNode, ctx: ctx}, nil +} \ No newline at end of file diff --git a/statediff/ipfs/dag_putter.go b/statediff/ipfs/dag_putter.go new file mode 100644 index 000000000000..92ffa1e8aea6 --- /dev/null +++ b/statediff/ipfs/dag_putter.go @@ -0,0 +1,79 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "bytes" + "encoding/gob" + "github.com/i-norden/go-ethereum/statediff" + + ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" +) + +const ( + EthStateDiffCode = 0x99 // Register custom codec for state diff? +) + +type DagPutter interface { + DagPut(sd *statediff.StateDiff) (string, error) +} + +type dagPutter struct { + Adder +} + +func NewDagPutter(adder Adder) *dagPutter { + return &dagPutter{Adder: adder} +} + +func (bhdp *dagPutter) DagPut(sd *statediff.StateDiff) (string, error) { + nd, err := bhdp.getNode(sd) + if err != nil { + return "", err + } + err = bhdp.Add(nd) + if err != nil { + return "", err + } + return nd.Cid().String(), nil +} + +func (bhdp *dagPutter) getNode(sd *statediff.StateDiff) (ipld.Node, error) { + + var buff bytes.Buffer + enc := gob.NewEncoder(&buff) + + err := enc.Encode(sd) + if err != nil { + return nil, err + } + + raw := buff.Bytes() + cid, err := RawToCid(EthStateDiffCode, raw) + if err != nil { + return nil, err + } + + return &StateDiffNode{ + StateDiff: sd, + cid: cid, + rawdata: raw, + }, nil +} \ No newline at end of file diff --git a/statediff/persister.go b/statediff/ipfs/helpers.go similarity index 71% rename from statediff/persister.go rename to statediff/ipfs/helpers.go index a3eba08260f9..a9904afa507d 100644 --- a/statediff/persister.go +++ b/statediff/ipfs/helpers.go @@ -17,24 +17,22 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff - -type Persister interface { - PersistStateDiff(sd *StateDiff) error -} - -type persister struct { - -} - -func NewPersister() *persister { - return &persister{ - +package ipfs + +import ( + mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" + "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" +) + +func RawToCid(codec uint64, raw []byte) (*cid.Cid, error) { + c, err := cid.Prefix{ + Codec: codec, + Version: 1, + MhType: mh.KECCAK_256, + MhLength: -1, + }.Sum(raw) + if err != nil { + return nil, err } -} - -func (p *persister) PersistStateDiff(sd *StateDiff) error { - //TODO: Persist state diff in IPFS - - return nil + return c, nil } \ No newline at end of file diff --git a/statediff/ipfs/node.go b/statediff/ipfs/node.go new file mode 100644 index 000000000000..dd7447a80ccf --- /dev/null +++ b/statediff/ipfs/node.go @@ -0,0 +1,78 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" + "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" + + "github.com/i-norden/go-ethereum/statediff" +) + +type StateDiffNode struct { + *statediff.StateDiff + + cid *cid.Cid + rawdata []byte +} + +func (sdn *StateDiffNode) RawData() []byte { + return sdn.rawdata +} + +func (sdn *StateDiffNode) Cid() *cid.Cid { + return sdn.cid +} + +func (sdn StateDiffNode) String() string { + return sdn.cid.String() +} + +func (sdn StateDiffNode) Loggable() map[string]interface{} { + return sdn.cid.Loggable() +} + +func (sdn StateDiffNode) Resolve(path []string) (interface{}, []string, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Tree(path string, depth int) []string { + panic("implement me") +} + +func (sdn StateDiffNode) ResolveLink(path []string) (*ipld.Link, []string, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Copy() ipld.Node { + panic("implement me") +} + +func (sdn StateDiffNode) Links() []*ipld.Link { + panic("implement me") +} + +func (sdn StateDiffNode) Stat() (*ipld.NodeStat, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Size() (uint64, error) { + panic("implement me") +} \ No newline at end of file diff --git a/statediff/publisher.go b/statediff/publisher.go new file mode 100644 index 000000000000..163f8d3b0e4a --- /dev/null +++ b/statediff/publisher.go @@ -0,0 +1,63 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "errors" + "github.com/i-norden/go-ethereum/statediff/ipfs" +) + +type Publisher interface { + PublishStateDiff(sd *StateDiff) (string, error) +} + +type publisher struct { + ipfs.DagPutter + Config +} + +func NewPublisher(config Config) (*publisher, error) { + adder, err := ipfs.NewAdder(config.Path) + if err != nil { + return nil, err + } + + return &publisher{ + DagPutter: ipfs.NewDagPutter(adder), + Config: config, + }, nil +} + +func (p *publisher) PublishStateDiff(sd *StateDiff) (string, error) { + switch p.Mode { + case IPLD: + cidStr, err := p.DagPut(sd) + if err != nil { + return "", err + } + + return cidStr, err + case LDB: + case SQL: + default: + } + + return "", errors.New("state diff publisher: unhandled publishing mode") +} \ No newline at end of file diff --git a/statediff/persister_test.go b/statediff/publisher_test.go similarity index 100% rename from statediff/persister_test.go rename to statediff/publisher_test.go diff --git a/statediff/statediff.go b/statediff/struct.go similarity index 98% rename from statediff/statediff.go rename to statediff/struct.go index f2e606936bbd..2142ec755c90 100644 --- a/statediff/statediff.go +++ b/statediff/struct.go @@ -58,7 +58,7 @@ func (sd *StateDiff) Encode() ([]byte, error) { type AccountDiffEventual struct { Nonce diffUint64 `json:"nonce" gencodec:"required"` Balance diffBigInt `json:"balance" gencodec:"required"` - Code string `json:"code" gencodec:"required"` + Code []byte `json:"code" gencodec:"required"` CodeHash string `json:"codeHash" gencodec:"required"` ContractRoot diffString `json:"contractRoot" gencodec:"required"` Storage map[string]diffString `json:"storage" gencodec:"required"` From 7957c5a92b19dde4ad440cc0720c60373fb3b46c Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:15:19 -0600 Subject: [PATCH 04/11] Add a state diff service --- cmd/geth/usage.go | 6 +++++ cmd/utils/flags.go | 13 +++++++++++ statediff/service.go | 54 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 statediff/service.go diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 82f17e0ee895..152ac059b0c0 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -245,6 +245,12 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerLegacyExtraDataFlag, }, }, + { + Name: "STATE DIFF", + Flags: []cli.Flag{ + utils.StateDiffFlag, + }, + }, { Name: "MISC", }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c18d98383fcb..b16f9cdcacd1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -58,6 +58,7 @@ import ( "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -1338,6 +1339,18 @@ func RegisterEthStatsService(stack *node.Node, url string) { } } +func RegisterStateDiffService(stack *node.Node) { + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + var ethServ *eth.Ethereum + ctx.Service(ðServ) + chainDb := ethServ.ChainDb() + blockChain := ethServ.BlockChain() + return statediff.NewStateDiffService(chainDb, blockChain) + }); err != nil { + Fatalf("Failed to register State Diff Service", err) + } +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/statediff/service.go b/statediff/service.go new file mode 100644 index 000000000000..695e3d638712 --- /dev/null +++ b/statediff/service.go @@ -0,0 +1,54 @@ +package statediff + +import ( + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/ethdb" + "fmt" +) + +type StateDiffService struct { + builder *builder + extractor *extractor + blockchain *core.BlockChain +} + +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { + config := Config{} + extractor, _ := NewExtractor(db, config) + return &StateDiffService{ + blockchain: blockChain, + extractor: extractor, + }, nil +} + +func (StateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} + +} + +func (StateDiffService) APIs() []rpc.API { + return []rpc.API{} +} + +func (sds *StateDiffService) Start(server *p2p.Server) error { + fmt.Println("starting the state diff service") + blockChannel := make(chan core.ChainHeadEvent) + sds.blockchain.SubscribeChainHeadEvent(blockChannel) + for { + select { + case <-blockChannel: + headOfChainEvent := <-blockChannel + previousBlock := headOfChainEvent.Block + //TODO: figure out the best way to get the previous block + currentBlock := headOfChainEvent.Block + sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) + } + } + return nil +} +func (StateDiffService) Stop() error { + fmt.Println("stopping the state diff service") + return nil +} \ No newline at end of file From 6fd20597a77fa0a1a5d0fd83d9c6bb82542589bc Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:20:25 -0600 Subject: [PATCH 05/11] Remove diff extractor from blockchain --- core/blockchain.go | 19 ------------------- eth/backend.go | 1 - 2 files changed, 20 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 77295b888cf2..1b48020a6c15 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -44,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" - "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -73,7 +72,6 @@ type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - StateDiff statediff.Config // Settings for state diff extraction } // BlockChain represents the canonical chain given a database with a genesis @@ -135,8 +133,6 @@ type BlockChain struct { badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. - - diffExtractor statediff.Extractor // State diff processing interface } // NewBlockChain returns a fully initialised block chain using information @@ -178,12 +174,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) var err error - if cacheConfig.StateDiff.On { - bc.diffExtractor, err = statediff.NewExtractor(db, cacheConfig.StateDiff) - if err != nil { - return nil, err - } - } bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { @@ -1216,15 +1206,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } proctime := time.Since(bstart) - // If extracting statediffs, do so now - if bc.cacheConfig.StateDiff.On { - // Currently not doing anything with returned cid... - _, err = bc.diffExtractor.ExtractStateDiff(*parent, *block) - if err != nil { - return i, events, coalescedLogs, err - } - } - // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) if err != nil { diff --git a/eth/backend.go b/eth/backend.go index d9617564351f..0de0a19803f2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -159,7 +159,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout, - StateDiff: config.StateDiff.On, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) From 32a0a238216db3fd988bf42f818cefb9d6fe5c9c Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:51:16 -0600 Subject: [PATCH 06/11] Update imports --- statediff/ipfs/dag_putter.go | 3 ++- statediff/publisher.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/statediff/ipfs/dag_putter.go b/statediff/ipfs/dag_putter.go index 92ffa1e8aea6..4f99bdae37e2 100644 --- a/statediff/ipfs/dag_putter.go +++ b/statediff/ipfs/dag_putter.go @@ -22,9 +22,10 @@ package ipfs import ( "bytes" "encoding/gob" - "github.com/i-norden/go-ethereum/statediff" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/common" ) const ( diff --git a/statediff/publisher.go b/statediff/publisher.go index 163f8d3b0e4a..48ad7cf64550 100644 --- a/statediff/publisher.go +++ b/statediff/publisher.go @@ -21,7 +21,7 @@ package statediff import ( "errors" - "github.com/i-norden/go-ethereum/statediff/ipfs" + "github.com/ethereum/go-ethereum/statediff/ipfs" ) type Publisher interface { From 70d531dc1ef8418d1f03341891fbd06d20a7a93a Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 11:05:00 -0600 Subject: [PATCH 07/11] Move statediff on/off check to geth cmd config --- cmd/geth/config.go | 4 ++++ cmd/utils/flags.go | 11 ----------- eth/config.go | 4 ---- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index b0749d23291b..a05e78b112e6 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -178,6 +178,10 @@ func makeFullNode(ctx *cli.Context) *node.Node { if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, cfg.Ethstats.URL) } + + if ctx.GlobalBool(utils.StateDiffFlag.Name) { + utils.RegisterStateDiffService(stack) + } return stack } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b16f9cdcacd1..54e3dd83d1ba 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1134,13 +1134,6 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { } } -// Check if state diff flags are on and applies them to eth context -func setStateDiff(ctx *cli.Context, cfg *eth.Config) { - if ctx.GlobalBool(StateDiffFlag.Name) && cfg.NoPruning && cfg.SyncMode == downloader.FullSync { - cfg.StateDiff.On = true - } -} - // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags @@ -1176,10 +1169,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" - if ctx.GlobalIsSet(StateDiffFlag.Name) { - setStateDiff(ctx, cfg) - } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } diff --git a/eth/config.go b/eth/config.go index c675b6d08cf6..601f4735e33f 100644 --- a/eth/config.go +++ b/eth/config.go @@ -17,7 +17,6 @@ package eth import ( - "github.com/ethereum/go-ethereum/statediff" "math/big" "os" "os/user" @@ -129,9 +128,6 @@ type Config struct { EWASMInterpreter string // Type of the EVM interpreter ("" for default) EVMInterpreter string - - // Config for state diff building - StateDiff statediff.Config } type configMarshaling struct { From 71147861f70b808669f453550ee5bdf44db0610d Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 13:52:02 -0600 Subject: [PATCH 08/11] Update starting state diff service --- statediff/service.go | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/statediff/service.go b/statediff/service.go index 695e3d638712..6870ce04788d 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -1,11 +1,12 @@ package statediff import ( - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/ethdb" - "fmt" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/event" + "log" ) type StateDiffService struct { @@ -19,7 +20,7 @@ func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*State extractor, _ := NewExtractor(db, config) return &StateDiffService{ blockchain: blockChain, - extractor: extractor, + extractor: extractor, }, nil } @@ -32,23 +33,32 @@ func (StateDiffService) APIs() []rpc.API { return []rpc.API{} } -func (sds *StateDiffService) Start(server *p2p.Server) error { - fmt.Println("starting the state diff service") - blockChannel := make(chan core.ChainHeadEvent) - sds.blockchain.SubscribeChainHeadEvent(blockChannel) +func (sds *StateDiffService) loop (sub event.Subscription, events chan core.ChainHeadEvent) { + defer sub.Unsubscribe() + for { select { - case <-blockChannel: - headOfChainEvent := <-blockChannel - previousBlock := headOfChainEvent.Block + case ev, ok := <-events: + if !ok { + log.Fatalf("Error getting chain head event from subscription.") + } + log.Println("doing something with an event", ev) + previousBlock := ev.Block //TODO: figure out the best way to get the previous block - currentBlock := headOfChainEvent.Block + currentBlock := ev.Block sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) } } + +} +func (sds *StateDiffService) Start(server *p2p.Server) error { + events := make(chan core.ChainHeadEvent, 10) + sub := sds.blockchain.SubscribeChainHeadEvent(events) + + go sds.loop(sub, events) + return nil } func (StateDiffService) Stop() error { - fmt.Println("stopping the state diff service") return nil -} \ No newline at end of file +} From cf4ed9c2b378904a24cb42fe90d6af4e99a0aa66 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 14:29:24 -0600 Subject: [PATCH 09/11] Add debugging logs for creating diff --- statediff/builder.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/statediff/builder.go b/statediff/builder.go index 433353591f5f..edfb50ca8740 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -49,10 +49,13 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block // Generate tries for old and new states oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) if err != nil { + log.Debug("error creating oldTrie", err) + //getting this error: error creating oldTrie missing trie node ddfbb83966d870891aa47147269447a83564d1defaefad5f9844a3a3a2a08433 (path ) return nil, err } newTrie, err := trie.New(newStateRoot, sdb.trieDB) if err != nil { + log.Debug("error creating newTrie", err) return nil, err } @@ -61,6 +64,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt := newTrie.NodeIterator([]byte{}) creations, err := sdb.collectDiffNodes(oldIt, newIt) if err != nil { + log.Debug("error collecting creation diff nodes", err) return nil, err } @@ -69,6 +73,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt = newTrie.NodeIterator(make([]byte, 0)) deletions, err := sdb.collectDiffNodes(newIt, oldIt) if err != nil { + log.Debug("error collecting deletion diff nodes", err) return nil, err } @@ -80,14 +85,17 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block // Build and return the statediff updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { + log.Debug("error building diff incremental for updated", err) return nil, err } createdAccounts, err := sdb.buildDiffEventual(creations, true) if err != nil { + log.Debug("error building diff incremental for created", err) return nil, err } deletedAccounts, err := sdb.buildDiffEventual(deletions, false) if err != nil { + log.Debug("error building diff incremental for deleted", err) return nil, err } From 097a9071ccee5df4f5f3b2a63936d89f945b6577 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 3 Dec 2018 16:23:15 -0600 Subject: [PATCH 10/11] Add state diff extractor test and small refactoring WIP - Add state diff builder test Clean up builder test --- statediff/builder.go | 34 ++-- statediff/builder_test.go | 299 ++++++++++++++++++++++++++++++ statediff/extractor.go | 20 +- statediff/extractor_test.go | 81 +++++++- statediff/service.go | 8 +- statediff/statediff_suite_test.go | 13 ++ statediff/struct.go | 22 +-- statediff/testhelpers/mocks.go | 49 +++++ 8 files changed, 483 insertions(+), 43 deletions(-) create mode 100644 statediff/statediff_suite_test.go create mode 100644 statediff/testhelpers/mocks.go diff --git a/statediff/builder.go b/statediff/builder.go index edfb50ca8740..da58b3bf4b50 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -109,7 +109,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block } func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { - var diffAccounts map[common.Address]*state.Account + var diffAccounts = make(map[common.Address]*state.Account) it, _ := trie.NewDifferenceIterator(a, b) for { @@ -161,15 +161,15 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account hexRoot := val.Root.Hex() if created { - nonce := diffUint64{ + nonce := DiffUint64{ NewValue: &val.Nonce, } - balance := diffBigInt{ + balance := DiffBigInt{ NewValue: val.Balance, } - contractRoot := diffString{ + contractRoot := DiffString{ NewValue: &hexRoot, } accountDiffs[addr] = AccountDiffEventual{ @@ -181,13 +181,13 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account Storage: storageDiffs, } } else { - nonce := diffUint64{ + nonce := DiffUint64{ OldValue: &val.Nonce, } - balance := diffBigInt{ + balance := DiffBigInt{ OldValue: val.Balance, } - contractRoot := diffString{ + contractRoot := DiffString{ OldValue: &hexRoot, } accountDiffs[addr] = AccountDiffEventual{ @@ -214,12 +214,12 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc log.Error("Failed building storage diffs", "Address", val, "error", err) return nil, err } else { - nonce := diffUint64{ + nonce := DiffUint64{ NewValue: &createdAcc.Nonce, OldValue: &deletedAcc.Nonce, } - balance := diffBigInt{ + balance := DiffBigInt{ NewValue: createdAcc.Balance, OldValue: deletedAcc.Balance, } @@ -227,7 +227,7 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc nHexRoot := createdAcc.Root.Hex() oHexRoot := deletedAcc.Root.Hex() - contractRoot := diffString{ + contractRoot := DiffString{ NewValue: &nHexRoot, OldValue: &oHexRoot, } @@ -246,14 +246,14 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc return updatedAccounts, nil } -func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) { +func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]DiffString, error) { log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) sTrie, err := trie.New(sr, sdb.trieDB) if err != nil { return nil, err } it := sTrie.NodeIterator(make([]byte, 0)) - storageDiffs := make(map[string]diffString) + storageDiffs := make(map[string]DiffString) for { log.Debug("Iterating over state at path ", "path", pathToStr(it)) if it.Leaf() { @@ -261,9 +261,9 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (ma path := pathToStr(it) value := common.ToHex(it.LeafBlob()) if creation { - storageDiffs[path] = diffString{NewValue: &value} + storageDiffs[path] = DiffString{NewValue: &value} } else { - storageDiffs[path] = diffString{OldValue: &value} + storageDiffs[path] = DiffString{OldValue: &value} } } cont := it.Next(true) @@ -274,7 +274,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (ma return storageDiffs, nil } -func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) { +func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]DiffString, error) { log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) oldTrie, err := trie.New(oldSR, sdb.trieDB) if err != nil { @@ -288,7 +288,7 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common oldIt := oldTrie.NodeIterator(make([]byte, 0)) newIt := newTrie.NodeIterator(make([]byte, 0)) it, _ := trie.NewDifferenceIterator(oldIt, newIt) - storageDiffs := make(map[string]diffString) + storageDiffs := make(map[string]DiffString) for { if it.Leaf() { log.Debug("Found leaf in storage", "path", pathToStr(it)) @@ -298,7 +298,7 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common log.Error("Failed to look up value in oldTrie", "path", path, "error", err) } else { hexOldVal := common.ToHex(oldVal) - storageDiffs[path] = diffString{OldValue: &hexOldVal, NewValue: &value} + storageDiffs[path] = DiffString{OldValue: &hexOldVal, NewValue: &value} } } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 6433b167a44d..64986af10a48 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -19,3 +19,302 @@ package statediff_test +import ( + "github.com/onsi/ginkgo" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "math/big" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/statediff" + "github.com/onsi/gomega" +) + + +var ( + testdb = ethdb.NewMemDatabase() + + testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + testBankFunds = big.NewInt(100000000) + genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) + + account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + + contractCode = common.Hex2Bytes("606060405260cc8060106000396000f360606040526000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146041578063c16431b914606b57603f565b005b6055600480803590602001909190505060a9565b6040518082815260200191505060405180910390f35b60886004808035906020019091908035906020019091905050608a565b005b80600060005083606481101560025790900160005b50819055505b5050565b6000600060005082606481101560025790900160005b5054905060c7565b91905056") + contractAddr common.Address + + emptyAccountDiffEventualMap = make(map[common.Address]statediff.AccountDiffEventual) + emptyAccountDiffIncrementalMap = make(map[common.Address]statediff.AccountDiffIncremental) +) +/* +contract test { + + uint256[100] data; + + function Put(uint256 addr, uint256 value) { + data[addr] = value; + } + + function Get(uint256 addr) constant returns (uint256 value) { + return data[addr]; + } +} +*/ + + +// makeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 3rd block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { + blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, testChainGen) + hashes := make([]common.Hash, n+1) + hashes[len(hashes)-1] = parent.Hash() + blockm := make(map[common.Hash]*types.Block, n+1) + blockm[parent.Hash()] = parent + for i, b := range blocks { + hashes[len(hashes)-i-2] = b.Hash() + blockm[b.Hash()] = b + } + return hashes, blockm +} + + +func testChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // account1Addr passes it on to account #2. + // account1Addr creates a test contract. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + nonce := block.TxNonce(account1Addr) + tx2, _ := types.SignTx(types.NewTransaction(nonce, account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, account1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), contractCode), signer, account1Key) + contractAddr = crypto.CreateAddress(account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592 + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(account2Addr) + //get function: 60cd2685 + //put function: c16431b9 + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), contractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) + block.AddTx(tx) + } +} +var _ = ginkgo.FDescribe("", func() { + var ( + block0Hash, block1Hash, block2Hash, block3Hash common.Hash + block0, block1, block2, block3 *types.Block + builder statediff.Builder + miningReward = int64(3000000000000000000) + burnAddress = common.HexToAddress("0x0") + diff *statediff.StateDiff + err error + ) + + ginkgo.BeforeEach(func() { + _, blocks := makeChain(3, 0, genesis) + block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash = common.HexToHash("0x47c398dd688eaa4dd11b006888156783fe32df83d59b197c0fcd303408103d39") + block2Hash = common.HexToHash("0x351b2f531838683ba457e8ca4d3a844cc48147dceafbcb589dc6e3227856ee75") + block3Hash = common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73") + + block0 = blocks[block0Hash] + block1 = blocks[block1Hash] + block2 = blocks[block2Hash] + block3 = blocks[block3Hash] + builder = statediff.NewBuilder(testdb) + }) + + ginkgo.It("returns empty account diff collections when the state root hasn't changed", func() { + expectedDiff := statediff.StateDiff{ + BlockNumber: block0.Number().Int64(), + BlockHash: block0Hash, + CreatedAccounts: emptyAccountDiffEventualMap, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: emptyAccountDiffIncrementalMap, + } + + diff, err := builder.BuildStateDiff(block0.Root(), block0.Root(), block0.Number().Int64(), block0Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(diff).To(gomega.Equal(&expectedDiff)) + }) + + ginkgo.Context("Block 1", func() { + //10000 transferred from testBankAddress to account1Addr + var balanceChange = int64(10000) + + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block0.Root(), block1.Root(), block1.Number().Int64(), block1Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block1.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block1Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance diffs for updated accounts", func() { + expectedBankBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(testBankFunds.Int64() - balanceChange), + OldValue: testBankFunds, + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(1)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Balance).To(gomega.Equal(expectedBankBalanceDiff)) + }) + + ginkgo.It("returns balance diffs for new accounts", func() { + expectedAccount1BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(balanceChange), + OldValue: nil, + } + + expectedBurnAddrBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(miningReward), + OldValue: nil, + } + + gomega.Expect(len(diff.CreatedAccounts)).To(gomega.Equal(2)) + gomega.Expect(diff.CreatedAccounts[account1Addr].Balance).To(gomega.Equal(expectedAccount1BalanceDiff)) + gomega.Expect(diff.CreatedAccounts[burnAddress].Balance).To(gomega.Equal(expectedBurnAddrBalanceDiff)) + }) + }) + + ginkgo.Context("Block 2", func() { + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + var ( + balanceChange = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + ) + + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block1.Root(), block2.Root(), block2.Number().Int64(), block2Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block2.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block2Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance diffs for updated accounts", func() { + expectedBankBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block1BankBalance - balanceChange), + OldValue: big.NewInt(block1BankBalance), + } + + expectedAccount1BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block1Account1Balance - balanceChange + balanceChange), + OldValue: big.NewInt(block1Account1Balance), + } + + expectedBurnBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(miningReward + miningReward), + OldValue: big.NewInt(miningReward), + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(3)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Balance).To(gomega.Equal(expectedBankBalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[account1Addr].Balance).To(gomega.Equal(expectedAccount1BalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[burnAddress].Balance).To(gomega.Equal(expectedBurnBalanceDiff)) + }) + + ginkgo.It("returns balance diffs for new accounts", func() { + expectedAccount2BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(balanceChange), + OldValue: nil, + } + + expectedContractBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(0), + OldValue: nil, + } + + gomega.Expect(len(diff.CreatedAccounts)).To(gomega.Equal(2)) + gomega.Expect(diff.CreatedAccounts[account2Addr].Balance).To(gomega.Equal(expectedAccount2BalanceDiff)) + gomega.Expect(diff.CreatedAccounts[contractAddr].Balance).To(gomega.Equal(expectedContractBalanceDiff)) + }) + }) + + ginkgo.Context("Block 3", func() { + //the contract's storage is changed + //and the block is mined by account 2 + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block2.Root(), block3.Root(), block3.Number().Int64(), block3Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block3.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block3Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns an empty collection for created accounts", func() { + gomega.Expect(diff.CreatedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance, storage and nonce diffs for updated accounts", func() { + block2Account2Balance := int64(1000) + expectedAcct2BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block2Account2Balance + miningReward), + OldValue: big.NewInt(block2Account2Balance), + } + + expectedContractStorageDiff := make(map[string]statediff.DiffString) + newVal := "0x03" + oldVal := "0x0" + path := "0x405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace" + expectedContractStorageDiff[path] = statediff.DiffString{ + NewValue: &newVal, + OldValue: &oldVal, + } + + oldNonce := uint64(2) + newNonce := uint64(3) + expectedBankNonceDiff := statediff.DiffUint64{ + NewValue: &newNonce, + OldValue: &oldNonce, + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(3)) + gomega.Expect(diff.UpdatedAccounts[account2Addr].Balance).To(gomega.Equal(expectedAcct2BalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[contractAddr].Storage).To(gomega.Equal(expectedContractStorageDiff)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Nonce).To(gomega.Equal(expectedBankNonceDiff)) + }) + }) +}) \ No newline at end of file diff --git a/statediff/extractor.go b/statediff/extractor.go index 3cbb5a311604..9d9b1f080c37 100644 --- a/statediff/extractor.go +++ b/statediff/extractor.go @@ -21,7 +21,6 @@ package statediff import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" ) type Extractor interface { @@ -29,27 +28,22 @@ type Extractor interface { } type extractor struct { - *builder // Interface for building state diff objects from two blocks - *publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) + Builder Builder // Interface for building state diff objects from two blocks + Publisher Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) } -func NewExtractor(db ethdb.Database, config Config) (*extractor, error) { - publisher, err := NewPublisher(config) - if err != nil { - return nil, err - } - +func NewExtractor(builder Builder, publisher Publisher) (*extractor, error) { return &extractor{ - builder: NewBuilder(db), - publisher: publisher, + Builder: builder, + Publisher: publisher, }, nil } func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) { - stateDiff, err := e.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) + stateDiff, err := e.Builder.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) if err != nil { return "", err } - return e.PublishStateDiff(stateDiff) + return e.Publisher.PublishStateDiff(stateDiff) } \ No newline at end of file diff --git a/statediff/extractor_test.go b/statediff/extractor_test.go index de5e170957b0..9ccd250f6c95 100644 --- a/statediff/extractor_test.go +++ b/statediff/extractor_test.go @@ -17,4 +17,83 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff_test \ No newline at end of file +package statediff_test + +import ( + "github.com/onsi/ginkgo" + "github.com/ethereum/go-ethereum/statediff" + "github.com/onsi/gomega" + "github.com/ethereum/go-ethereum/core/types" + "math/rand" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + "math/big" +) +var _ = ginkgo.Describe("Extractor", func() { + var publisher testhelpers.MockPublisher + var builder testhelpers.MockBuilder + var currentBlockNumber *big.Int + var parentBlock, currentBlock *types.Block + var expectedStateDiff statediff.StateDiff + var extractor statediff.Extractor + var err error + + ginkgo.BeforeEach(func() { + publisher = testhelpers.MockPublisher{} + builder = testhelpers.MockBuilder{} + extractor, err = statediff.NewExtractor(&builder, &publisher) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + blockNumber := rand.Int63() + parentBlockNumber := big.NewInt(blockNumber - int64(1)) + currentBlockNumber = big.NewInt(blockNumber) + parentBlock = types.NewBlock(&types.Header{Number: parentBlockNumber}, nil, nil, nil) + currentBlock = types.NewBlock(&types.Header{Number: currentBlockNumber}, nil, nil, nil) + + expectedStateDiff = statediff.StateDiff{ + BlockNumber: blockNumber, + BlockHash: currentBlock.Hash(), + CreatedAccounts: nil, + DeletedAccounts: nil, + UpdatedAccounts: nil, + } + }) + + ginkgo.It("builds a state diff struct", func() { + builder.SetStateDiffToBuild(&expectedStateDiff) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(builder.OldStateRoot).To(gomega.Equal(parentBlock.Root())) + gomega.Expect(builder.NewStateRoot).To(gomega.Equal(currentBlock.Root())) + gomega.Expect(builder.BlockNumber).To(gomega.Equal(currentBlockNumber.Int64())) + gomega.Expect(builder.BlockHash).To(gomega.Equal(currentBlock.Hash())) + }) + + ginkgo.It("returns an error if building the state diff fails", func() { + builder.SetBuilderError(testhelpers.MockError) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err).To(gomega.MatchError(testhelpers.MockError)) + }) + + ginkgo.It("publishes the state diff struct", func() { + builder.SetStateDiffToBuild(&expectedStateDiff) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(publisher.StateDiff).To(gomega.Equal(&expectedStateDiff)) + }) + + ginkgo.It("returns an error if publishing the diff fails", func() { + publisher.SetPublisherError(testhelpers.MockError) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err).To(gomega.MatchError(testhelpers.MockError)) + }) +}) diff --git a/statediff/service.go b/statediff/service.go index 6870ce04788d..86839cf3b83b 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -17,7 +17,13 @@ type StateDiffService struct { func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { config := Config{} - extractor, _ := NewExtractor(db, config) + builder := NewBuilder(db) + publisher, err := NewPublisher(config) + if err != nil { + return nil, nil + } + + extractor, _ := NewExtractor(builder, publisher) return &StateDiffService{ blockchain: blockChain, extractor: extractor, diff --git a/statediff/statediff_suite_test.go b/statediff/statediff_suite_test.go new file mode 100644 index 000000000000..7118ea792291 --- /dev/null +++ b/statediff/statediff_suite_test.go @@ -0,0 +1,13 @@ +package statediff_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStatediff(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Statediff Suite") +} diff --git a/statediff/struct.go b/statediff/struct.go index 2142ec755c90..009af4c2ea6c 100644 --- a/statediff/struct.go +++ b/statediff/struct.go @@ -56,31 +56,31 @@ func (sd *StateDiff) Encode() ([]byte, error) { } type AccountDiffEventual struct { - Nonce diffUint64 `json:"nonce" gencodec:"required"` - Balance diffBigInt `json:"balance" gencodec:"required"` + Nonce DiffUint64 `json:"nonce" gencodec:"required"` + Balance DiffBigInt `json:"balance" gencodec:"required"` Code []byte `json:"code" gencodec:"required"` CodeHash string `json:"codeHash" gencodec:"required"` - ContractRoot diffString `json:"contractRoot" gencodec:"required"` - Storage map[string]diffString `json:"storage" gencodec:"required"` + ContractRoot DiffString `json:"contractRoot" gencodec:"required"` + Storage map[string]DiffString `json:"storage" gencodec:"required"` } type AccountDiffIncremental struct { - Nonce diffUint64 `json:"nonce" gencodec:"required"` - Balance diffBigInt `json:"balance" gencodec:"required"` + Nonce DiffUint64 `json:"nonce" gencodec:"required"` + Balance DiffBigInt `json:"balance" gencodec:"required"` CodeHash string `json:"codeHash" gencodec:"required"` - ContractRoot diffString `json:"contractRoot" gencodec:"required"` - Storage map[string]diffString `json:"storage" gencodec:"required"` + ContractRoot DiffString `json:"contractRoot" gencodec:"required"` + Storage map[string]DiffString `json:"storage" gencodec:"required"` } -type diffString struct { +type DiffString struct { NewValue *string `json:"newValue" gencodec:"optional"` OldValue *string `json:"oldValue" gencodec:"optional"` } -type diffUint64 struct { +type DiffUint64 struct { NewValue *uint64 `json:"newValue" gencodec:"optional"` OldValue *uint64 `json:"oldValue" gencodec:"optional"` } -type diffBigInt struct { +type DiffBigInt struct { NewValue *big.Int `json:"newValue" gencodec:"optional"` OldValue *big.Int `json:"oldValue" gencodec:"optional"` } diff --git a/statediff/testhelpers/mocks.go b/statediff/testhelpers/mocks.go new file mode 100644 index 000000000000..4bff3c02862d --- /dev/null +++ b/statediff/testhelpers/mocks.go @@ -0,0 +1,49 @@ +package testhelpers + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff" + "errors" +) + +var MockError = errors.New("mock error") + +type MockBuilder struct { + OldStateRoot common.Hash + NewStateRoot common.Hash + BlockNumber int64 + BlockHash common.Hash + stateDiff *statediff.StateDiff + builderError error +} + +func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*statediff.StateDiff, error) { + builder.OldStateRoot = oldStateRoot + builder.NewStateRoot = newStateRoot + builder.BlockNumber = blockNumber + builder.BlockHash = blockHash + + return builder.stateDiff, builder.builderError +} + +func (builder *MockBuilder) SetStateDiffToBuild(stateDiff *statediff.StateDiff) { + builder.stateDiff = stateDiff +} + +func (builder *MockBuilder) SetBuilderError(err error) { + builder.builderError = err +} + +type MockPublisher struct{ + StateDiff *statediff.StateDiff + publisherError error +} + +func (publisher *MockPublisher) PublishStateDiff(sd *statediff.StateDiff) (string, error) { + publisher.StateDiff = sd + return "", publisher.publisherError +} + +func (publisher *MockPublisher) SetPublisherError(err error) { + publisher.publisherError = err +} From c1e3af009348391aa93ce0322dea0facdd5d203b Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 12 Dec 2018 11:54:51 -0600 Subject: [PATCH 11/11] reorganize to fix unallowed import cycles between statediff and statediff/ipfs --- statediff/{ => builder}/builder.go | 2 +- statediff/builder/builder_suite_test.go | 32 +++++++ statediff/{ => builder}/builder_test.go | 10 +- statediff/{ => builder}/helpers.go | 2 +- statediff/{ => builder}/struct.go | 2 +- statediff/{ => extractor}/extractor.go | 14 +-- statediff/extractor/extractor_suite_test.go | 32 +++++++ statediff/{ => extractor}/extractor_test.go | 23 +++-- statediff/{ => publisher}/ipfs/adder.go | 2 +- statediff/{ => publisher}/ipfs/dag_putter.go | 12 +-- statediff/{ => publisher}/ipfs/helpers.go | 4 +- statediff/{ => publisher}/ipfs/node.go | 12 +-- statediff/{ => publisher}/publisher.go | 21 +++-- statediff/publisher/publisher_suite_test.go | 32 +++++++ statediff/publisher/publisher_test.go | 20 ++++ statediff/service.go | 70 -------------- statediff/service/service.go | 94 +++++++++++++++++++ statediff/service/service_suite_test.go | 32 +++++++ statediff/service/service_test.go | 20 ++++ ..._test.go => statediff_integration_test.go} | 0 statediff/statediff_suite_test.go | 23 ++++- statediff/testhelpers/mocks.go | 33 +++++-- 22 files changed, 366 insertions(+), 126 deletions(-) rename statediff/{ => builder}/builder.go (99%) create mode 100644 statediff/builder/builder_suite_test.go rename statediff/{ => builder}/builder_test.go (99%) rename statediff/{ => builder}/helpers.go (99%) rename statediff/{ => builder}/struct.go (99%) rename statediff/{ => extractor}/extractor.go (77%) create mode 100644 statediff/extractor/extractor_suite_test.go rename statediff/{ => extractor}/extractor_test.go (87%) rename statediff/{ => publisher}/ipfs/adder.go (96%) rename statediff/{ => publisher}/ipfs/dag_putter.go (82%) rename statediff/{ => publisher}/ipfs/helpers.go (94%) rename statediff/{ => publisher}/ipfs/node.go (87%) rename statediff/{ => publisher}/publisher.go (75%) create mode 100644 statediff/publisher/publisher_suite_test.go create mode 100644 statediff/publisher/publisher_test.go delete mode 100644 statediff/service.go create mode 100644 statediff/service/service.go create mode 100644 statediff/service/service_suite_test.go create mode 100644 statediff/service/service_test.go rename statediff/{publisher_test.go => statediff_integration_test.go} (100%) diff --git a/statediff/builder.go b/statediff/builder/builder.go similarity index 99% rename from statediff/builder.go rename to statediff/builder/builder.go index da58b3bf4b50..30479a60cfe0 100644 --- a/statediff/builder.go +++ b/statediff/builder/builder.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package builder import ( "github.com/ethereum/go-ethereum/common" diff --git a/statediff/builder/builder_suite_test.go b/statediff/builder/builder_suite_test.go new file mode 100644 index 000000000000..bbbf63078e7a --- /dev/null +++ b/statediff/builder/builder_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiffBuilder(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Builder Suite") +} \ No newline at end of file diff --git a/statediff/builder_test.go b/statediff/builder/builder_test.go similarity index 99% rename from statediff/builder_test.go rename to statediff/builder/builder_test.go index 64986af10a48..793a6347d419 100644 --- a/statediff/builder_test.go +++ b/statediff/builder/builder_test.go @@ -17,20 +17,22 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff_test +package builder_test import ( + "math/big" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" - "math/big" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/statediff" - "github.com/onsi/gomega" + statediff "github.com/ethereum/go-ethereum/statediff/builder" ) diff --git a/statediff/helpers.go b/statediff/builder/helpers.go similarity index 99% rename from statediff/helpers.go rename to statediff/builder/helpers.go index 8a0a6466e172..31aae926bfaf 100644 --- a/statediff/helpers.go +++ b/statediff/builder/helpers.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package builder import ( "sort" diff --git a/statediff/struct.go b/statediff/builder/struct.go similarity index 99% rename from statediff/struct.go rename to statediff/builder/struct.go index 009af4c2ea6c..7b48bba72ccf 100644 --- a/statediff/struct.go +++ b/statediff/builder/struct.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package builder import ( "encoding/json" diff --git a/statediff/extractor.go b/statediff/extractor/extractor.go similarity index 77% rename from statediff/extractor.go rename to statediff/extractor/extractor.go index 9d9b1f080c37..d485f5e13067 100644 --- a/statediff/extractor.go +++ b/statediff/extractor/extractor.go @@ -17,10 +17,12 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package extractor import ( "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/publisher" ) type Extractor interface { @@ -28,14 +30,14 @@ type Extractor interface { } type extractor struct { - Builder Builder // Interface for building state diff objects from two blocks - Publisher Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) + Builder builder.Builder // Interface for building state diff objects from two blocks + Publisher publisher.Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) } -func NewExtractor(builder Builder, publisher Publisher) (*extractor, error) { +func NewExtractor(b builder.Builder, p publisher.Publisher) (*extractor, error) { return &extractor{ - Builder: builder, - Publisher: publisher, + Builder: b, + Publisher: p, }, nil } diff --git a/statediff/extractor/extractor_suite_test.go b/statediff/extractor/extractor_suite_test.go new file mode 100644 index 000000000000..fe7719032769 --- /dev/null +++ b/statediff/extractor/extractor_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package extractor_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiffExtractor(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Extractor Suite") +} \ No newline at end of file diff --git a/statediff/extractor_test.go b/statediff/extractor/extractor_test.go similarity index 87% rename from statediff/extractor_test.go rename to statediff/extractor/extractor_test.go index 9ccd250f6c95..d76329726452 100644 --- a/statediff/extractor_test.go +++ b/statediff/extractor/extractor_test.go @@ -17,16 +17,19 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff_test +package extractor_test import ( + "math/big" + "math/rand" + "github.com/onsi/ginkgo" - "github.com/ethereum/go-ethereum/statediff" "github.com/onsi/gomega" + + "github.com/ethereum/go-ethereum/statediff/extractor" + statediff "github.com/ethereum/go-ethereum/statediff/builder" "github.com/ethereum/go-ethereum/core/types" - "math/rand" "github.com/ethereum/go-ethereum/statediff/testhelpers" - "math/big" ) var _ = ginkgo.Describe("Extractor", func() { var publisher testhelpers.MockPublisher @@ -34,13 +37,13 @@ var _ = ginkgo.Describe("Extractor", func() { var currentBlockNumber *big.Int var parentBlock, currentBlock *types.Block var expectedStateDiff statediff.StateDiff - var extractor statediff.Extractor + var ex extractor.Extractor var err error ginkgo.BeforeEach(func() { publisher = testhelpers.MockPublisher{} builder = testhelpers.MockBuilder{} - extractor, err = statediff.NewExtractor(&builder, &publisher) + ex, err = extractor.NewExtractor(&builder, &publisher) gomega.Expect(err).NotTo(gomega.HaveOccurred()) blockNumber := rand.Int63() @@ -61,7 +64,7 @@ var _ = ginkgo.Describe("Extractor", func() { ginkgo.It("builds a state diff struct", func() { builder.SetStateDiffToBuild(&expectedStateDiff) - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + _, err = ex.ExtractStateDiff(*parentBlock, *currentBlock) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(builder.OldStateRoot).To(gomega.Equal(parentBlock.Root())) @@ -73,7 +76,7 @@ var _ = ginkgo.Describe("Extractor", func() { ginkgo.It("returns an error if building the state diff fails", func() { builder.SetBuilderError(testhelpers.MockError) - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + _, err = ex.ExtractStateDiff(*parentBlock, *currentBlock) gomega.Expect(err).To(gomega.HaveOccurred()) gomega.Expect(err).To(gomega.MatchError(testhelpers.MockError)) @@ -82,7 +85,7 @@ var _ = ginkgo.Describe("Extractor", func() { ginkgo.It("publishes the state diff struct", func() { builder.SetStateDiffToBuild(&expectedStateDiff) - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + _, err = ex.ExtractStateDiff(*parentBlock, *currentBlock) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(publisher.StateDiff).To(gomega.Equal(&expectedStateDiff)) @@ -91,7 +94,7 @@ var _ = ginkgo.Describe("Extractor", func() { ginkgo.It("returns an error if publishing the diff fails", func() { publisher.SetPublisherError(testhelpers.MockError) - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + _, err = ex.ExtractStateDiff(*parentBlock, *currentBlock) gomega.Expect(err).To(gomega.HaveOccurred()) gomega.Expect(err).To(gomega.MatchError(testhelpers.MockError)) diff --git a/statediff/ipfs/adder.go b/statediff/publisher/ipfs/adder.go similarity index 96% rename from statediff/ipfs/adder.go rename to statediff/publisher/ipfs/adder.go index 23eea1c1ff54..eb8e012c53a1 100644 --- a/statediff/ipfs/adder.go +++ b/statediff/publisher/ipfs/adder.go @@ -24,7 +24,7 @@ import ( "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/repo/fsrepo" - ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" ) type Adder interface { diff --git a/statediff/ipfs/dag_putter.go b/statediff/publisher/ipfs/dag_putter.go similarity index 82% rename from statediff/ipfs/dag_putter.go rename to statediff/publisher/ipfs/dag_putter.go index 4f99bdae37e2..2f2ff02e2eef 100644 --- a/statediff/ipfs/dag_putter.go +++ b/statediff/publisher/ipfs/dag_putter.go @@ -23,9 +23,9 @@ import ( "bytes" "encoding/gob" - ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/common" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + + "github.com/ethereum/go-ethereum/statediff/builder" ) const ( @@ -33,7 +33,7 @@ const ( ) type DagPutter interface { - DagPut(sd *statediff.StateDiff) (string, error) + DagPut(sd *builder.StateDiff) (string, error) } type dagPutter struct { @@ -44,7 +44,7 @@ func NewDagPutter(adder Adder) *dagPutter { return &dagPutter{Adder: adder} } -func (bhdp *dagPutter) DagPut(sd *statediff.StateDiff) (string, error) { +func (bhdp *dagPutter) DagPut(sd *builder.StateDiff) (string, error) { nd, err := bhdp.getNode(sd) if err != nil { return "", err @@ -56,7 +56,7 @@ func (bhdp *dagPutter) DagPut(sd *statediff.StateDiff) (string, error) { return nd.Cid().String(), nil } -func (bhdp *dagPutter) getNode(sd *statediff.StateDiff) (ipld.Node, error) { +func (bhdp *dagPutter) getNode(sd *builder.StateDiff) (ipld.Node, error) { var buff bytes.Buffer enc := gob.NewEncoder(&buff) diff --git a/statediff/ipfs/helpers.go b/statediff/publisher/ipfs/helpers.go similarity index 94% rename from statediff/ipfs/helpers.go rename to statediff/publisher/ipfs/helpers.go index a9904afa507d..ea6ca5ab5c78 100644 --- a/statediff/ipfs/helpers.go +++ b/statediff/publisher/ipfs/helpers.go @@ -21,7 +21,7 @@ package ipfs import ( mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" - "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" ) func RawToCid(codec uint64, raw []byte) (*cid.Cid, error) { @@ -34,5 +34,5 @@ func RawToCid(codec uint64, raw []byte) (*cid.Cid, error) { if err != nil { return nil, err } - return c, nil + return &c, nil } \ No newline at end of file diff --git a/statediff/ipfs/node.go b/statediff/publisher/ipfs/node.go similarity index 87% rename from statediff/ipfs/node.go rename to statediff/publisher/ipfs/node.go index dd7447a80ccf..1d8daa5e89a0 100644 --- a/statediff/ipfs/node.go +++ b/statediff/publisher/ipfs/node.go @@ -20,14 +20,14 @@ package ipfs import ( - ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" - "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" - "github.com/i-norden/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" ) type StateDiffNode struct { - *statediff.StateDiff + *builder.StateDiff cid *cid.Cid rawdata []byte @@ -37,8 +37,8 @@ func (sdn *StateDiffNode) RawData() []byte { return sdn.rawdata } -func (sdn *StateDiffNode) Cid() *cid.Cid { - return sdn.cid +func (sdn *StateDiffNode) Cid() cid.Cid { + return *sdn.cid } func (sdn StateDiffNode) String() string { diff --git a/statediff/publisher.go b/statediff/publisher/publisher.go similarity index 75% rename from statediff/publisher.go rename to statediff/publisher/publisher.go index 48ad7cf64550..404871405ff2 100644 --- a/statediff/publisher.go +++ b/statediff/publisher/publisher.go @@ -17,23 +17,26 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package publisher import ( "errors" - "github.com/ethereum/go-ethereum/statediff/ipfs" + + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/publisher/ipfs" ) type Publisher interface { - PublishStateDiff(sd *StateDiff) (string, error) + PublishStateDiff(sd *builder.StateDiff) (string, error) } type publisher struct { ipfs.DagPutter - Config + statediff.Config } -func NewPublisher(config Config) (*publisher, error) { +func NewPublisher(config statediff.Config) (*publisher, error) { adder, err := ipfs.NewAdder(config.Path) if err != nil { return nil, err @@ -45,17 +48,17 @@ func NewPublisher(config Config) (*publisher, error) { }, nil } -func (p *publisher) PublishStateDiff(sd *StateDiff) (string, error) { +func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { switch p.Mode { - case IPLD: + case statediff.IPLD: cidStr, err := p.DagPut(sd) if err != nil { return "", err } return cidStr, err - case LDB: - case SQL: + case statediff.LDB: + case statediff.SQL: default: } diff --git a/statediff/publisher/publisher_suite_test.go b/statediff/publisher/publisher_suite_test.go new file mode 100644 index 000000000000..29d10e2a105c --- /dev/null +++ b/statediff/publisher/publisher_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package publisher_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiffPublisher(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Publisher Suite") +} \ No newline at end of file diff --git a/statediff/publisher/publisher_test.go b/statediff/publisher/publisher_test.go new file mode 100644 index 000000000000..520163ddc476 --- /dev/null +++ b/statediff/publisher/publisher_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package publisher_test \ No newline at end of file diff --git a/statediff/service.go b/statediff/service.go deleted file mode 100644 index 86839cf3b83b..000000000000 --- a/statediff/service.go +++ /dev/null @@ -1,70 +0,0 @@ -package statediff - -import ( - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/event" - "log" -) - -type StateDiffService struct { - builder *builder - extractor *extractor - blockchain *core.BlockChain -} - -func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { - config := Config{} - builder := NewBuilder(db) - publisher, err := NewPublisher(config) - if err != nil { - return nil, nil - } - - extractor, _ := NewExtractor(builder, publisher) - return &StateDiffService{ - blockchain: blockChain, - extractor: extractor, - }, nil -} - -func (StateDiffService) Protocols() []p2p.Protocol { - return []p2p.Protocol{} - -} - -func (StateDiffService) APIs() []rpc.API { - return []rpc.API{} -} - -func (sds *StateDiffService) loop (sub event.Subscription, events chan core.ChainHeadEvent) { - defer sub.Unsubscribe() - - for { - select { - case ev, ok := <-events: - if !ok { - log.Fatalf("Error getting chain head event from subscription.") - } - log.Println("doing something with an event", ev) - previousBlock := ev.Block - //TODO: figure out the best way to get the previous block - currentBlock := ev.Block - sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) - } - } - -} -func (sds *StateDiffService) Start(server *p2p.Server) error { - events := make(chan core.ChainHeadEvent, 10) - sub := sds.blockchain.SubscribeChainHeadEvent(events) - - go sds.loop(sub, events) - - return nil -} -func (StateDiffService) Stop() error { - return nil -} diff --git a/statediff/service/service.go b/statediff/service/service.go new file mode 100644 index 000000000000..b3da7532d796 --- /dev/null +++ b/statediff/service/service.go @@ -0,0 +1,94 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package service + +import ( + "log" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/extractor" + "github.com/ethereum/go-ethereum/statediff/publisher" +) + +type StateDiffService struct { + builder builder.Builder + extractor extractor.Extractor + blockchain *core.BlockChain +} + +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { + config := statediff.Config{} + b := builder.NewBuilder(db) + p, err := publisher.NewPublisher(config) + if err != nil { + return nil, nil + } + + e, _ := extractor.NewExtractor(b, p) + return &StateDiffService{ + blockchain: blockChain, + extractor: e, + }, nil +} + +func (StateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} + +} + +func (StateDiffService) APIs() []rpc.API { + return []rpc.API{} +} + +func (sds *StateDiffService) loop (sub event.Subscription, events chan core.ChainHeadEvent) { + defer sub.Unsubscribe() + + for { + select { + case ev, ok := <-events: + if !ok { + log.Fatalf("Error getting chain head event from subscription.") + } + log.Println("doing something with an event", ev) + previousBlock := ev.Block + //TODO: figure out the best way to get the previous block + currentBlock := ev.Block + sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) + } + } + +} +func (sds *StateDiffService) Start(server *p2p.Server) error { + events := make(chan core.ChainHeadEvent, 10) + sub := sds.blockchain.SubscribeChainHeadEvent(events) + + go sds.loop(sub, events) + + return nil +} +func (StateDiffService) Stop() error { + return nil +} diff --git a/statediff/service/service_suite_test.go b/statediff/service/service_suite_test.go new file mode 100644 index 000000000000..60212a8fd617 --- /dev/null +++ b/statediff/service/service_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package service_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiffService(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Service Suite") +} \ No newline at end of file diff --git a/statediff/service/service_test.go b/statediff/service/service_test.go new file mode 100644 index 000000000000..99e6cfd639a3 --- /dev/null +++ b/statediff/service/service_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package service_test \ No newline at end of file diff --git a/statediff/publisher_test.go b/statediff/statediff_integration_test.go similarity index 100% rename from statediff/publisher_test.go rename to statediff/statediff_integration_test.go diff --git a/statediff/statediff_suite_test.go b/statediff/statediff_suite_test.go index 7118ea792291..99834db4810e 100644 --- a/statediff/statediff_suite_test.go +++ b/statediff/statediff_suite_test.go @@ -1,3 +1,22 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + package statediff_test import ( @@ -7,7 +26,7 @@ import ( . "github.com/onsi/gomega" ) -func TestStatediff(t *testing.T) { +func TestStateDiff(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Statediff Suite") + RunSpecs(t, "StateDiff Suite") } diff --git a/statediff/testhelpers/mocks.go b/statediff/testhelpers/mocks.go index 4bff3c02862d..b4608e455391 100644 --- a/statediff/testhelpers/mocks.go +++ b/statediff/testhelpers/mocks.go @@ -1,9 +1,28 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + package testhelpers import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff" "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/builder" ) var MockError = errors.New("mock error") @@ -13,11 +32,11 @@ type MockBuilder struct { NewStateRoot common.Hash BlockNumber int64 BlockHash common.Hash - stateDiff *statediff.StateDiff + stateDiff *builder.StateDiff builderError error } -func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*statediff.StateDiff, error) { +func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*builder.StateDiff, error) { builder.OldStateRoot = oldStateRoot builder.NewStateRoot = newStateRoot builder.BlockNumber = blockNumber @@ -26,7 +45,7 @@ func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Has return builder.stateDiff, builder.builderError } -func (builder *MockBuilder) SetStateDiffToBuild(stateDiff *statediff.StateDiff) { +func (builder *MockBuilder) SetStateDiffToBuild(stateDiff *builder.StateDiff) { builder.stateDiff = stateDiff } @@ -35,11 +54,11 @@ func (builder *MockBuilder) SetBuilderError(err error) { } type MockPublisher struct{ - StateDiff *statediff.StateDiff + StateDiff *builder.StateDiff publisherError error } -func (publisher *MockPublisher) PublishStateDiff(sd *statediff.StateDiff) (string, error) { +func (publisher *MockPublisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { publisher.StateDiff = sd return "", publisher.publisherError }