From 53a6b1c6892b13cbda56c6b8fad001df47e2ebb0 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 21 Nov 2018 18:44:57 -0600 Subject: [PATCH 01/13] port statediff from https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go; minor fixes --- statediff/helpers.go | 114 +++++++++++++ statediff/statediff.go | 67 ++++++++ statediff/statediff_builder.go | 301 +++++++++++++++++++++++++++++++++ 3 files changed, 482 insertions(+) create mode 100644 statediff/helpers.go create mode 100644 statediff/statediff.go create mode 100644 statediff/statediff_builder.go diff --git a/statediff/helpers.go b/statediff/helpers.go new file mode 100644 index 000000000000..976d21fe2896 --- /dev/null +++ b/statediff/helpers.go @@ -0,0 +1,114 @@ +package statediff + +import ( + "sort" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/trie" +) + +func sortKeys(data map[common.Address]*state.Account) []string { + var keys []string + for key := range data { + keys = append(keys, key.Hex()) + } + sort.Strings(keys) + + return keys +} + +func findIntersection(a, b []string) []string { + lenA := len(a) + lenB := len(b) + iOfA, iOfB := 0, 0 + updates := make([]string, 0) + if iOfA >= lenA || iOfB >= lenB { + return updates + } + for { + switch strings.Compare(a[iOfA], b[iOfB]) { + // a[iOfA] < b[iOfB] + case -1: + iOfA++ + if iOfA >= lenA { + return updates + } + break + // a[iOfA] == b[iOfB] + case 0: + updates = append(updates, a[iOfA]) + iOfA++ + iOfB++ + if iOfA >= lenA || iOfB >= lenB { + return updates + } + break + // a[iOfA] > b[iOfB] + case 1: + iOfB++ + if iOfB >= lenB { + return updates + } + break + } + } + +} + +func pathToStr(it trie.NodeIterator) string { + path := it.Path() + if hasTerm(path) { + path = path[:len(path)-1] + } + nibblePath := "" + for i, v := range common.ToHex(path) { + if i%2 == 0 && i > 1 { + continue + } + nibblePath = nibblePath + string(v) + } + + return nibblePath +} + +// Duplicated from trie/encoding.go +func hexToKeybytes(hex []byte) []byte { + if hasTerm(hex) { + hex = hex[:len(hex)-1] + } + if len(hex)&1 != 0 { + panic("can't convert hex key of odd length") + } + key := make([]byte, (len(hex)+1)/2) + decodeNibbles(hex, key) + + return key +} + +func decodeNibbles(nibbles []byte, bytes []byte) { + for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { + bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] + } +} + +// prefixLen returns the length of the common prefix of a and b. +func prefixLen(a, b []byte) int { + var i, length = 0, len(a) + if len(b) < length { + length = len(b) + } + for ; i < length; i++ { + if a[i] != b[i] { + break + } + } + + return i +} + +// hasTerm returns whether a hex key has the terminator flag. +func hasTerm(s []byte) bool { + return len(s) > 0 && s[len(s)-1] == 16 +} diff --git a/statediff/statediff.go b/statediff/statediff.go new file mode 100644 index 000000000000..d980ef867d34 --- /dev/null +++ b/statediff/statediff.go @@ -0,0 +1,67 @@ +package statediff + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +type StateDiff struct { + BlockNumber int64 `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts map[common.Address]AccountDiffEventual `json:"createdAccounts" gencodec:"required"` + DeletedAccounts map[common.Address]AccountDiffEventual `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts map[common.Address]AccountDiffIncremental `json:"updatedAccounts" gencodec:"required"` + + encoded []byte + err error +} + +func (self *StateDiff) ensureEncoded() { + if self.encoded == nil && self.err == nil { + self.encoded, self.err = json.Marshal(self) + } +} + +// Implement Encoder interface for StateDiff +func (sd *StateDiff) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Implement Encoder interface for StateDiff +func (sd *StateDiff) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err +} + +type AccountDiffEventual struct { + Nonce diffUint64 `json:"nonce" gencodec:"required"` + Balance diffBigInt `json:"balance" gencodec:"required"` + Code string `json:"code" gencodec:"required"` + CodeHash string `json:"codeHash" gencodec:"required"` + ContractRoot diffString `json:"contractRoot" gencodec:"required"` + Storage map[string]diffString `json:"storage" gencodec:"required"` +} + +type AccountDiffIncremental struct { + Nonce diffUint64 `json:"nonce" gencodec:"required"` + Balance diffBigInt `json:"balance" gencodec:"required"` + CodeHash string `json:"codeHash" gencodec:"required"` + ContractRoot diffString `json:"contractRoot" gencodec:"required"` + Storage map[string]diffString `json:"storage" gencodec:"required"` +} + +type diffString struct { + NewValue *string `json:"newValue" gencodec:"optional"` + OldValue *string `json:"oldValue" gencodec:"optional"` +} +type diffUint64 struct { + NewValue *uint64 `json:"newValue" gencodec:"optional"` + OldValue *uint64 `json:"oldValue" gencodec:"optional"` +} +type diffBigInt struct { + NewValue *big.Int `json:"newValue" gencodec:"optional"` + OldValue *big.Int `json:"oldValue" gencodec:"optional"` +} diff --git a/statediff/statediff_builder.go b/statediff/statediff_builder.go new file mode 100644 index 000000000000..c3011ed66239 --- /dev/null +++ b/statediff/statediff_builder.go @@ -0,0 +1,301 @@ +package statediff + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +type StateDiffBuilder interface { + CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) +} + +type stateDiffBuilder struct { + chainDB ethdb.Database + trieDB *trie.Database + cachedTrie *trie.Trie +} + +func NewStateDiffBuilder(db ethdb.Database) *stateDiffBuilder { + return &stateDiffBuilder{ + chainDB: db, + trieDB: trie.NewDatabase(db), + } +} + +func (sdb *stateDiffBuilder) CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { + // Generate tries for old and new states + oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) + if err != nil { + return nil, err + } + newTrie, err := trie.New(newStateRoot, sdb.trieDB) + if err != nil { + return nil, err + } + + // Find created accounts + oldIt := oldTrie.NodeIterator([]byte{}) + newIt := newTrie.NodeIterator([]byte{}) + creations, err := sdb.collectDiffNodes(oldIt, newIt) + if err != nil { + return nil, err + } + + // Find deleted accounts + oldIt = oldTrie.NodeIterator(make([]byte, 0)) + newIt = newTrie.NodeIterator(make([]byte, 0)) + deletions, err := sdb.collectDiffNodes(newIt, oldIt) + if err != nil { + return nil, err + } + + // Find all the diffed keys + createKeys := sortKeys(creations) + deleteKeys := sortKeys(deletions) + updatedKeys := findIntersection(createKeys, deleteKeys) + + // Build and return the statediff + updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, &updatedKeys) + if err != nil { + return nil, err + } + createdAccounts, err := sdb.buildDiffEventual(creations, true) + if err != nil { + return nil, err + } + deletedAccounts, err := sdb.buildDiffEventual(deletions, false) + if err != nil { + return nil, err + } + + return &StateDiff{ + BlockNumber: blockNumber, + BlockHash: blockHash, + CreatedAccounts: createdAccounts, + DeletedAccounts: deletedAccounts, + UpdatedAccounts: updatedAccounts, + }, nil +} + +func (sdb *stateDiffBuilder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { + var diffAccounts map[common.Address]*state.Account + it, _ := trie.NewDifferenceIterator(a, b) + + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", common.Hash(it.Hash())) + if it.Leaf() { + + // lookup address + path := make([]byte, len(it.Path())-1) + copy(path, it.Path()) + addr, err := sdb.addressByPath(path) + if err != nil { + log.Error("Error looking up address via path", "path", path, "error", err) + return nil, err + } + + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil { + log.Error("Error looking up account via address", "address", addr, "error", err) + return nil, err + } + + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", addr, "account", account) + diffAccounts[*addr] = &account + } + cont := it.Next(true) + if !cont { + break + } + } + return diffAccounts, nil +} + +func (sdb *stateDiffBuilder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) { + accountDiffs := make(map[common.Address]AccountDiffEventual) + for addr, val := range accounts { + sr := val.Root + if storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created); err != nil { + log.Error("Failed building eventual storage diffs", "Address", val, "error", err) + return nil, err + } else { + code := "" + codeBytes, err := sdb.chainDB.Get(val.CodeHash) + if err == nil && len(codeBytes) != 0 { + code = common.ToHex(codeBytes) + } else { + log.Debug("No code field.", "codehash", val.CodeHash, "Address", val, "error", err) + } + codeHash := common.ToHex(val.CodeHash) + if created { + nonce := diffUint64{ + NewValue: &val.Nonce, + } + + balance := diffBigInt{ + NewValue: val.Balance, + } + + hexRoot := val.Root.Hex() + contractRoot := diffString{ + NewValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + Code: code, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + } else { + nonce := diffUint64{ + OldValue: &val.Nonce, + } + balance := diffBigInt{ + OldValue: val.Balance, + } + hexRoot := val.Root.Hex() + contractRoot := diffString{ + OldValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + } + } + } + return accountDiffs, nil +} + +func (sdb *stateDiffBuilder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) { + updatedAccounts := make(map[common.Address]AccountDiffIncremental) + for _, val := range *updatedKeys { + createdAcc := creations[common.HexToAddress(val)] + deletedAcc := deletions[common.HexToAddress(val)] + oldSR := deletedAcc.Root + newSR := createdAcc.Root + if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil { + log.Error("Failed building storage diffs", "Address", val, "error", err) + return nil, err + } else { + nonce := diffUint64{ + NewValue: &createdAcc.Nonce, + OldValue: &deletedAcc.Nonce, + } + + balance := diffBigInt{ + NewValue: createdAcc.Balance, + OldValue: deletedAcc.Balance, + } + codeHash := common.ToHex(createdAcc.CodeHash) + + nHexRoot := createdAcc.Root.Hex() + oHexRoot := deletedAcc.Root.Hex() + contractRoot := diffString{ + NewValue: &nHexRoot, + OldValue: &oHexRoot, + } + + updatedAccounts[common.HexToAddress(val)] = AccountDiffIncremental{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + delete(creations, common.HexToAddress(val)) + delete(deletions, common.HexToAddress(val)) + } + } + return updatedAccounts, nil +} + +func (sdb *stateDiffBuilder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) { + log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) + sTrie, err := trie.New(sr, sdb.trieDB) + if err != nil { + return nil, err + } + it := sTrie.NodeIterator(make([]byte, 0)) + storageDiffs := make(map[string]diffString) + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + path := pathToStr(it) + value := common.ToHex(it.LeafBlob()) + if creation { + storageDiffs[path] = diffString{NewValue: &value} + } else { + storageDiffs[path] = diffString{OldValue: &value} + } + } + cont := it.Next(true) + if !cont { + break + } + } + return storageDiffs, nil +} + +func (sdb *stateDiffBuilder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) { + log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) + oldTrie, err := trie.New(oldSR, sdb.trieDB) + if err != nil { + return nil, err + } + newTrie, err := trie.New(newSR, sdb.trieDB) + if err != nil { + return nil, err + } + + oldIt := oldTrie.NodeIterator(make([]byte, 0)) + newIt := newTrie.NodeIterator(make([]byte, 0)) + it, _ := trie.NewDifferenceIterator(oldIt, newIt) + storageDiffs := make(map[string]diffString) + for { + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + path := pathToStr(it) + value := common.ToHex(it.LeafBlob()) + if oldVal, err := oldTrie.TryGet(it.LeafKey()); err != nil { + log.Error("Failed to look up value in oldTrie", "path", path, "error", err) + } else { + hexOldVal := common.ToHex(oldVal) + storageDiffs[path] = diffString{OldValue: &hexOldVal, NewValue: &value} + } + } + + cont := it.Next(true) + if !cont { + break + } + } + return storageDiffs, nil +} + +func (sdb *stateDiffBuilder) addressByPath(path []byte) (*common.Address, error) { + // db := core.PreimageTable(sdb.chainDb) + log.Debug("Looking up address from path", "path", common.ToHex(append([]byte("secure-key-"), path...))) + // if addrBytes,err := db.Get(path); err != nil { + if addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeybytes(path)...)); err != nil { + log.Error("Error looking up address via path", "path", common.ToHex(append([]byte("secure-key-"), path...)), "error", err) + return nil, err + } else { + addr := common.BytesToAddress(addrBytes) + log.Debug("Address found", "Address", addr) + return &addr, nil + } + +} From 26e8088cc5f676bcdbd51c889a8303d4603cefd8 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sun, 25 Nov 2018 23:52:53 -0600 Subject: [PATCH 02/13] integrating state diff extracting, building, and persisting into geth processes --- cmd/geth/main.go | 1 + cmd/utils/flags.go | 17 ++++ core/blockchain.go | 14 ++++ eth/backend.go | 8 +- eth/config.go | 4 + .../{statediff_builder.go => builder.go} | 43 +++++++--- statediff/builder_test.go | 21 +++++ statediff/config.go | 80 +++++++++++++++++++ statediff/extractor.go | 50 ++++++++++++ statediff/extractor_test.go | 20 +++++ statediff/helpers.go | 19 +++++ statediff/persister.go | 40 ++++++++++ statediff/persister_test.go | 20 +++++ statediff/statediff.go | 19 +++++ 14 files changed, 343 insertions(+), 13 deletions(-) rename statediff/{statediff_builder.go => builder.go} (78%) create mode 100644 statediff/builder_test.go create mode 100644 statediff/config.go create mode 100644 statediff/extractor.go create mode 100644 statediff/extractor_test.go create mode 100644 statediff/persister.go create mode 100644 statediff/persister_test.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 69802a48a763..15f76554851c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -132,6 +132,7 @@ var ( utils.GpoPercentileFlag, utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, + utils.StateDiffFlag, configFileFlag, } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d7b698c7e2e5..c18d98383fcb 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -626,6 +626,12 @@ var ( Usage: "External EVM configuration (default = built-in interpreter)", Value: "", } + + // Statediff flags + StateDiffFlag = cli.BoolFlag{ + Name: "statediff", + Usage: "Enables the calculation of state diffs between each block, persists these state diffs in ipfs", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1127,6 +1133,13 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { } } +// Check if state diff flags are on and applies them to eth context +func setStateDiff(ctx *cli.Context, cfg *eth.Config) { + if ctx.GlobalBool(StateDiffFlag.Name) && cfg.NoPruning && cfg.SyncMode == downloader.FullSync { + cfg.StateDiff.On = true + } +} + // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags @@ -1162,6 +1175,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" + if ctx.GlobalIsSet(StateDiffFlag.Name) { + setStateDiff(ctx, cfg) + } + if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } diff --git a/core/blockchain.go b/core/blockchain.go index d173b2de294d..fed6b4b74dae 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -20,6 +20,7 @@ package core import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/statediff" "io" "math/big" mrand "math/rand" @@ -72,6 +73,7 @@ type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + StateDiff bool // Whether or not to calculate and persist state diffs } // BlockChain represents the canonical chain given a database with a genesis @@ -133,6 +135,8 @@ type BlockChain struct { badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. + + diffExtractor statediff.Extractor // State diff processing interface } // NewBlockChain returns a fully initialised block chain using information @@ -173,6 +177,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) + if cacheConfig.StateDiff { + bc.diffExtractor = statediff.NewExtractor(db) + } + var err error bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { @@ -1187,6 +1195,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty parent = chain[i-1] } state, err := state.New(parent.Root(), bc.stateCache) + if err != nil { return i, events, coalescedLogs, err } @@ -1204,6 +1213,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } proctime := time.Since(bstart) + // If extracting statediffs, do so now + if bc.cacheConfig.StateDiff { + bc.diffExtractor.Extract(*parent, *block) + } + // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) if err != nil { diff --git a/eth/backend.go b/eth/backend.go index 4721408425cb..d9617564351f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -154,7 +154,13 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { EWASMInterpreter: config.EWASMInterpreter, EVMInterpreter: config.EVMInterpreter, } - cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout} + cacheConfig = &core.CacheConfig{ + Disabled: config.NoPruning, + TrieCleanLimit: config.TrieCleanCache, + TrieDirtyLimit: config.TrieDirtyCache, + TrieTimeLimit: config.TrieTimeout, + StateDiff: config.StateDiff.On, + } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) if err != nil { diff --git a/eth/config.go b/eth/config.go index 601f4735e33f..c675b6d08cf6 100644 --- a/eth/config.go +++ b/eth/config.go @@ -17,6 +17,7 @@ package eth import ( + "github.com/ethereum/go-ethereum/statediff" "math/big" "os" "os/user" @@ -128,6 +129,9 @@ type Config struct { EWASMInterpreter string // Type of the EVM interpreter ("" for default) EVMInterpreter string + + // Config for state diff building + StateDiff statediff.Config } type configMarshaling struct { diff --git a/statediff/statediff_builder.go b/statediff/builder.go similarity index 78% rename from statediff/statediff_builder.go rename to statediff/builder.go index c3011ed66239..9017c240dd67 100644 --- a/statediff/statediff_builder.go +++ b/statediff/builder.go @@ -1,3 +1,22 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + package statediff import ( @@ -9,24 +28,24 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -type StateDiffBuilder interface { - CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) +type Builder interface { + BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) } -type stateDiffBuilder struct { +type builder struct { chainDB ethdb.Database trieDB *trie.Database cachedTrie *trie.Trie } -func NewStateDiffBuilder(db ethdb.Database) *stateDiffBuilder { - return &stateDiffBuilder{ +func NewBuilder(db ethdb.Database) *builder { + return &builder{ chainDB: db, trieDB: trie.NewDatabase(db), } } -func (sdb *stateDiffBuilder) CreateStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { +func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { // Generate tries for old and new states oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) if err != nil { @@ -81,7 +100,7 @@ func (sdb *stateDiffBuilder) CreateStateDiff(oldStateRoot, newStateRoot common.H }, nil } -func (sdb *stateDiffBuilder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { +func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { var diffAccounts map[common.Address]*state.Account it, _ := trie.NewDifferenceIterator(a, b) @@ -117,7 +136,7 @@ func (sdb *stateDiffBuilder) collectDiffNodes(a, b trie.NodeIterator) (map[commo return diffAccounts, nil } -func (sdb *stateDiffBuilder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) { +func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) { accountDiffs := make(map[common.Address]AccountDiffEventual) for addr, val := range accounts { sr := val.Root @@ -178,7 +197,7 @@ func (sdb *stateDiffBuilder) buildDiffEventual(accounts map[common.Address]*stat return accountDiffs, nil } -func (sdb *stateDiffBuilder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) { +func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) { updatedAccounts := make(map[common.Address]AccountDiffIncremental) for _, val := range *updatedKeys { createdAcc := creations[common.HexToAddress(val)] @@ -221,7 +240,7 @@ func (sdb *stateDiffBuilder) buildDiffIncremental(creations map[common.Address]* return updatedAccounts, nil } -func (sdb *stateDiffBuilder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) { +func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) { log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) sTrie, err := trie.New(sr, sdb.trieDB) if err != nil { @@ -249,7 +268,7 @@ func (sdb *stateDiffBuilder) buildStorageDiffsEventual(sr common.Hash, creation return storageDiffs, nil } -func (sdb *stateDiffBuilder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) { +func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) { log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) oldTrie, err := trie.New(oldSR, sdb.trieDB) if err != nil { @@ -285,7 +304,7 @@ func (sdb *stateDiffBuilder) buildStorageDiffsIncremental(oldSR common.Hash, new return storageDiffs, nil } -func (sdb *stateDiffBuilder) addressByPath(path []byte) (*common.Address, error) { +func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { // db := core.PreimageTable(sdb.chainDb) log.Debug("Looking up address from path", "path", common.ToHex(append([]byte("secure-key-"), path...))) // if addrBytes,err := db.Get(path); err != nil { diff --git a/statediff/builder_test.go b/statediff/builder_test.go new file mode 100644 index 000000000000..6433b167a44d --- /dev/null +++ b/statediff/builder_test.go @@ -0,0 +1,21 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff_test + diff --git a/statediff/config.go b/statediff/config.go new file mode 100644 index 000000000000..ee31f271ecb9 --- /dev/null +++ b/statediff/config.go @@ -0,0 +1,80 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import "fmt" + +type Config struct { + On bool + Mode StateDiffMode +} + +type StateDiffMode int + +const ( + IPFS StateDiffMode = iota + LDB + SQL +) + +func (mode StateDiffMode) IsValid() bool { + return mode >= IPFS && mode <= SQL +} + +// String implements the stringer interface. +func (mode StateDiffMode) String() string { + switch mode { + case IPFS: + return "ipfs" + case LDB: + return "ldb" + case SQL: + return "sql" + default: + return "unknown" + } +} + +func (mode StateDiffMode) MarshalText() ([]byte, error) { + switch mode { + case IPFS: + return []byte("ipfs"), nil + case LDB: + return []byte("ldb"), nil + case SQL: + return []byte("sql"), nil + default: + return nil, fmt.Errorf("unknown state diff storage mode %d", mode) + } +} + +func (mode *StateDiffMode) UnmarshalText(text []byte) error { + switch string(text) { + case "ipfs": + *mode = IPFS + case "ldb": + *mode = LDB + case "sql": + *mode = SQL + default: + return fmt.Errorf(`unknown state diff storage mode %q, want "ipfs", "ldb" or "sql"`, text) + } + return nil +} \ No newline at end of file diff --git a/statediff/extractor.go b/statediff/extractor.go new file mode 100644 index 000000000000..2abe3ebb3b49 --- /dev/null +++ b/statediff/extractor.go @@ -0,0 +1,50 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" +) + +type Extractor interface { + ExtractStateDiff(parent, current types.Block) error +} + +type extractor struct { + b *builder + p *persister +} + +func NewExtractor(db ethdb.Database) *extractor { + return &extractor{ + b: NewBuilder(db), + p: NewPersister(), + } +} + +func (e *extractor) Extract(parent, current types.Block) error { + stateDiff, err := e.b.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) + if err != nil { + return err + } + + return e.p.PersistStateDiff(stateDiff) +} \ No newline at end of file diff --git a/statediff/extractor_test.go b/statediff/extractor_test.go new file mode 100644 index 000000000000..de5e170957b0 --- /dev/null +++ b/statediff/extractor_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff_test \ No newline at end of file diff --git a/statediff/helpers.go b/statediff/helpers.go index 976d21fe2896..50626ab2c4ca 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -1,3 +1,22 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + package statediff import ( diff --git a/statediff/persister.go b/statediff/persister.go new file mode 100644 index 000000000000..a3eba08260f9 --- /dev/null +++ b/statediff/persister.go @@ -0,0 +1,40 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +type Persister interface { + PersistStateDiff(sd *StateDiff) error +} + +type persister struct { + +} + +func NewPersister() *persister { + return &persister{ + + } +} + +func (p *persister) PersistStateDiff(sd *StateDiff) error { + //TODO: Persist state diff in IPFS + + return nil +} \ No newline at end of file diff --git a/statediff/persister_test.go b/statediff/persister_test.go new file mode 100644 index 000000000000..de5e170957b0 --- /dev/null +++ b/statediff/persister_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff_test \ No newline at end of file diff --git a/statediff/statediff.go b/statediff/statediff.go index d980ef867d34..f2e606936bbd 100644 --- a/statediff/statediff.go +++ b/statediff/statediff.go @@ -1,3 +1,22 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + package statediff import ( From 976bff614a201ff33af0bc0286fb568c20b4c725 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 26 Nov 2018 02:37:25 -0600 Subject: [PATCH 03/13] work towards persisting created statediffs in ipfs; based off github.com/vulcanize/eth-block-extractor --- core/blockchain.go | 21 ++-- statediff/builder.go | 98 +++++++++---------- statediff/config.go | 15 +-- statediff/extractor.go | 27 ++--- statediff/helpers.go | 17 +--- statediff/ipfs/adder.go | 58 +++++++++++ statediff/ipfs/dag_putter.go | 79 +++++++++++++++ statediff/{persister.go => ipfs/helpers.go} | 36 ++++--- statediff/ipfs/node.go | 78 +++++++++++++++ statediff/publisher.go | 63 ++++++++++++ .../{persister_test.go => publisher_test.go} | 0 statediff/{statediff.go => struct.go} | 2 +- 12 files changed, 383 insertions(+), 111 deletions(-) create mode 100644 statediff/ipfs/adder.go create mode 100644 statediff/ipfs/dag_putter.go rename statediff/{persister.go => ipfs/helpers.go} (71%) create mode 100644 statediff/ipfs/node.go create mode 100644 statediff/publisher.go rename statediff/{persister_test.go => publisher_test.go} (100%) rename statediff/{statediff.go => struct.go} (98%) diff --git a/core/blockchain.go b/core/blockchain.go index fed6b4b74dae..77295b888cf2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -20,7 +20,6 @@ package core import ( "errors" "fmt" - "github.com/ethereum/go-ethereum/statediff" "io" "math/big" mrand "math/rand" @@ -45,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" + "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -73,7 +73,7 @@ type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - StateDiff bool // Whether or not to calculate and persist state diffs + StateDiff statediff.Config // Settings for state diff extraction } // BlockChain represents the canonical chain given a database with a genesis @@ -177,11 +177,14 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) - if cacheConfig.StateDiff { - bc.diffExtractor = statediff.NewExtractor(db) + var err error + if cacheConfig.StateDiff.On { + bc.diffExtractor, err = statediff.NewExtractor(db, cacheConfig.StateDiff) + if err != nil { + return nil, err + } } - var err error bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { return nil, err @@ -1214,8 +1217,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty proctime := time.Since(bstart) // If extracting statediffs, do so now - if bc.cacheConfig.StateDiff { - bc.diffExtractor.Extract(*parent, *block) + if bc.cacheConfig.StateDiff.On { + // Currently not doing anything with returned cid... + _, err = bc.diffExtractor.ExtractStateDiff(*parent, *block) + if err != nil { + return i, events, coalescedLogs, err + } } // Write the block to the chain and get the status. diff --git a/statediff/builder.go b/statediff/builder.go index 9017c240dd67..433353591f5f 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -78,7 +78,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block updatedKeys := findIntersection(createKeys, deleteKeys) // Build and return the statediff - updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, &updatedKeys) + updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { return nil, err } @@ -133,6 +133,7 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address break } } + return diffAccounts, nil } @@ -140,66 +141,63 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account accountDiffs := make(map[common.Address]AccountDiffEventual) for addr, val := range accounts { sr := val.Root - if storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created); err != nil { + storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created) + if err != nil { log.Error("Failed building eventual storage diffs", "Address", val, "error", err) return nil, err + } + + codeBytes, err := sdb.chainDB.Get(val.CodeHash) + + codeHash := common.ToHex(val.CodeHash) + hexRoot := val.Root.Hex() + + if created { + nonce := diffUint64{ + NewValue: &val.Nonce, + } + + balance := diffBigInt{ + NewValue: val.Balance, + } + + contractRoot := diffString{ + NewValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + Code: codeBytes, + ContractRoot: contractRoot, + Storage: storageDiffs, + } } else { - code := "" - codeBytes, err := sdb.chainDB.Get(val.CodeHash) - if err == nil && len(codeBytes) != 0 { - code = common.ToHex(codeBytes) - } else { - log.Debug("No code field.", "codehash", val.CodeHash, "Address", val, "error", err) + nonce := diffUint64{ + OldValue: &val.Nonce, } - codeHash := common.ToHex(val.CodeHash) - if created { - nonce := diffUint64{ - NewValue: &val.Nonce, - } - - balance := diffBigInt{ - NewValue: val.Balance, - } - - hexRoot := val.Root.Hex() - contractRoot := diffString{ - NewValue: &hexRoot, - } - accountDiffs[addr] = AccountDiffEventual{ - Nonce: nonce, - Balance: balance, - CodeHash: codeHash, - Code: code, - ContractRoot: contractRoot, - Storage: storageDiffs, - } - } else { - nonce := diffUint64{ - OldValue: &val.Nonce, - } - balance := diffBigInt{ - OldValue: val.Balance, - } - hexRoot := val.Root.Hex() - contractRoot := diffString{ - OldValue: &hexRoot, - } - accountDiffs[addr] = AccountDiffEventual{ - Nonce: nonce, - Balance: balance, - CodeHash: codeHash, - ContractRoot: contractRoot, - Storage: storageDiffs, - } + balance := diffBigInt{ + OldValue: val.Balance, + } + contractRoot := diffString{ + OldValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, } } } + return accountDiffs, nil } -func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys *[]string) (map[common.Address]AccountDiffIncremental, error) { +func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys []string) (map[common.Address]AccountDiffIncremental, error) { updatedAccounts := make(map[common.Address]AccountDiffIncremental) - for _, val := range *updatedKeys { + for _, val := range updatedKeys { createdAcc := creations[common.HexToAddress(val)] deletedAcc := deletions[common.HexToAddress(val)] oldSR := deletedAcc.Root diff --git a/statediff/config.go b/statediff/config.go index ee31f271ecb9..ac207acd1347 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -22,26 +22,27 @@ package statediff import "fmt" type Config struct { - On bool - Mode StateDiffMode + On bool // Whether or not to extract state diffs + Mode StateDiffMode // Mode for storing diffs + Path string // Path for storing diffs } type StateDiffMode int const ( - IPFS StateDiffMode = iota + IPLD StateDiffMode = iota LDB SQL ) func (mode StateDiffMode) IsValid() bool { - return mode >= IPFS && mode <= SQL + return mode >= IPLD && mode <= SQL } // String implements the stringer interface. func (mode StateDiffMode) String() string { switch mode { - case IPFS: + case IPLD: return "ipfs" case LDB: return "ldb" @@ -54,7 +55,7 @@ func (mode StateDiffMode) String() string { func (mode StateDiffMode) MarshalText() ([]byte, error) { switch mode { - case IPFS: + case IPLD: return []byte("ipfs"), nil case LDB: return []byte("ldb"), nil @@ -68,7 +69,7 @@ func (mode StateDiffMode) MarshalText() ([]byte, error) { func (mode *StateDiffMode) UnmarshalText(text []byte) error { switch string(text) { case "ipfs": - *mode = IPFS + *mode = IPLD case "ldb": *mode = LDB case "sql": diff --git a/statediff/extractor.go b/statediff/extractor.go index 2abe3ebb3b49..3cbb5a311604 100644 --- a/statediff/extractor.go +++ b/statediff/extractor.go @@ -25,26 +25,31 @@ import ( ) type Extractor interface { - ExtractStateDiff(parent, current types.Block) error + ExtractStateDiff(parent, current types.Block) (string, error) } type extractor struct { - b *builder - p *persister + *builder // Interface for building state diff objects from two blocks + *publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) } -func NewExtractor(db ethdb.Database) *extractor { - return &extractor{ - b: NewBuilder(db), - p: NewPersister(), +func NewExtractor(db ethdb.Database, config Config) (*extractor, error) { + publisher, err := NewPublisher(config) + if err != nil { + return nil, err } + + return &extractor{ + builder: NewBuilder(db), + publisher: publisher, + }, nil } -func (e *extractor) Extract(parent, current types.Block) error { - stateDiff, err := e.b.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) +func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) { + stateDiff, err := e.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) if err != nil { - return err + return "", err } - return e.p.PersistStateDiff(stateDiff) + return e.PublishStateDiff(stateDiff) } \ No newline at end of file diff --git a/statediff/helpers.go b/statediff/helpers.go index 50626ab2c4ca..8a0a6466e172 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -112,22 +112,7 @@ func decodeNibbles(nibbles []byte, bytes []byte) { } } -// prefixLen returns the length of the common prefix of a and b. -func prefixLen(a, b []byte) int { - var i, length = 0, len(a) - if len(b) < length { - length = len(b) - } - for ; i < length; i++ { - if a[i] != b[i] { - break - } - } - - return i -} - // hasTerm returns whether a hex key has the terminator flag. func hasTerm(s []byte) bool { return len(s) > 0 && s[len(s)-1] == 16 -} +} \ No newline at end of file diff --git a/statediff/ipfs/adder.go b/statediff/ipfs/adder.go new file mode 100644 index 000000000000..23eea1c1ff54 --- /dev/null +++ b/statediff/ipfs/adder.go @@ -0,0 +1,58 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "context" + + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/repo/fsrepo" + ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" +) + +type Adder interface { + Add(node ipld.Node) error +} + +type adder struct { + n *core.IpfsNode + ctx context.Context +} + +func (a adder) Add(node ipld.Node) error { + return a.n.DAG.Add(a.n.Context(), node) // For some reason DAG.Add method is not being exposed by the ipld.DAGService +} + +func NewAdder(repoPath string) (*adder, error) { + r, err := fsrepo.Open(repoPath) + if err != nil { + return nil, err + } + ctx := context.Background() + cfg := &core.BuildCfg{ + Online: false, + Repo: r, + } + ipfsNode, err := core.NewNode(ctx, cfg) + if err != nil { + return nil, err + } + return &adder{n: ipfsNode, ctx: ctx}, nil +} \ No newline at end of file diff --git a/statediff/ipfs/dag_putter.go b/statediff/ipfs/dag_putter.go new file mode 100644 index 000000000000..92ffa1e8aea6 --- /dev/null +++ b/statediff/ipfs/dag_putter.go @@ -0,0 +1,79 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "bytes" + "encoding/gob" + "github.com/i-norden/go-ethereum/statediff" + + ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" +) + +const ( + EthStateDiffCode = 0x99 // Register custom codec for state diff? +) + +type DagPutter interface { + DagPut(sd *statediff.StateDiff) (string, error) +} + +type dagPutter struct { + Adder +} + +func NewDagPutter(adder Adder) *dagPutter { + return &dagPutter{Adder: adder} +} + +func (bhdp *dagPutter) DagPut(sd *statediff.StateDiff) (string, error) { + nd, err := bhdp.getNode(sd) + if err != nil { + return "", err + } + err = bhdp.Add(nd) + if err != nil { + return "", err + } + return nd.Cid().String(), nil +} + +func (bhdp *dagPutter) getNode(sd *statediff.StateDiff) (ipld.Node, error) { + + var buff bytes.Buffer + enc := gob.NewEncoder(&buff) + + err := enc.Encode(sd) + if err != nil { + return nil, err + } + + raw := buff.Bytes() + cid, err := RawToCid(EthStateDiffCode, raw) + if err != nil { + return nil, err + } + + return &StateDiffNode{ + StateDiff: sd, + cid: cid, + rawdata: raw, + }, nil +} \ No newline at end of file diff --git a/statediff/persister.go b/statediff/ipfs/helpers.go similarity index 71% rename from statediff/persister.go rename to statediff/ipfs/helpers.go index a3eba08260f9..a9904afa507d 100644 --- a/statediff/persister.go +++ b/statediff/ipfs/helpers.go @@ -17,24 +17,22 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff - -type Persister interface { - PersistStateDiff(sd *StateDiff) error -} - -type persister struct { - -} - -func NewPersister() *persister { - return &persister{ - +package ipfs + +import ( + mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" + "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" +) + +func RawToCid(codec uint64, raw []byte) (*cid.Cid, error) { + c, err := cid.Prefix{ + Codec: codec, + Version: 1, + MhType: mh.KECCAK_256, + MhLength: -1, + }.Sum(raw) + if err != nil { + return nil, err } -} - -func (p *persister) PersistStateDiff(sd *StateDiff) error { - //TODO: Persist state diff in IPFS - - return nil + return c, nil } \ No newline at end of file diff --git a/statediff/ipfs/node.go b/statediff/ipfs/node.go new file mode 100644 index 000000000000..dd7447a80ccf --- /dev/null +++ b/statediff/ipfs/node.go @@ -0,0 +1,78 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" + "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" + + "github.com/i-norden/go-ethereum/statediff" +) + +type StateDiffNode struct { + *statediff.StateDiff + + cid *cid.Cid + rawdata []byte +} + +func (sdn *StateDiffNode) RawData() []byte { + return sdn.rawdata +} + +func (sdn *StateDiffNode) Cid() *cid.Cid { + return sdn.cid +} + +func (sdn StateDiffNode) String() string { + return sdn.cid.String() +} + +func (sdn StateDiffNode) Loggable() map[string]interface{} { + return sdn.cid.Loggable() +} + +func (sdn StateDiffNode) Resolve(path []string) (interface{}, []string, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Tree(path string, depth int) []string { + panic("implement me") +} + +func (sdn StateDiffNode) ResolveLink(path []string) (*ipld.Link, []string, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Copy() ipld.Node { + panic("implement me") +} + +func (sdn StateDiffNode) Links() []*ipld.Link { + panic("implement me") +} + +func (sdn StateDiffNode) Stat() (*ipld.NodeStat, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Size() (uint64, error) { + panic("implement me") +} \ No newline at end of file diff --git a/statediff/publisher.go b/statediff/publisher.go new file mode 100644 index 000000000000..163f8d3b0e4a --- /dev/null +++ b/statediff/publisher.go @@ -0,0 +1,63 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "errors" + "github.com/i-norden/go-ethereum/statediff/ipfs" +) + +type Publisher interface { + PublishStateDiff(sd *StateDiff) (string, error) +} + +type publisher struct { + ipfs.DagPutter + Config +} + +func NewPublisher(config Config) (*publisher, error) { + adder, err := ipfs.NewAdder(config.Path) + if err != nil { + return nil, err + } + + return &publisher{ + DagPutter: ipfs.NewDagPutter(adder), + Config: config, + }, nil +} + +func (p *publisher) PublishStateDiff(sd *StateDiff) (string, error) { + switch p.Mode { + case IPLD: + cidStr, err := p.DagPut(sd) + if err != nil { + return "", err + } + + return cidStr, err + case LDB: + case SQL: + default: + } + + return "", errors.New("state diff publisher: unhandled publishing mode") +} \ No newline at end of file diff --git a/statediff/persister_test.go b/statediff/publisher_test.go similarity index 100% rename from statediff/persister_test.go rename to statediff/publisher_test.go diff --git a/statediff/statediff.go b/statediff/struct.go similarity index 98% rename from statediff/statediff.go rename to statediff/struct.go index f2e606936bbd..2142ec755c90 100644 --- a/statediff/statediff.go +++ b/statediff/struct.go @@ -58,7 +58,7 @@ func (sd *StateDiff) Encode() ([]byte, error) { type AccountDiffEventual struct { Nonce diffUint64 `json:"nonce" gencodec:"required"` Balance diffBigInt `json:"balance" gencodec:"required"` - Code string `json:"code" gencodec:"required"` + Code []byte `json:"code" gencodec:"required"` CodeHash string `json:"codeHash" gencodec:"required"` ContractRoot diffString `json:"contractRoot" gencodec:"required"` Storage map[string]diffString `json:"storage" gencodec:"required"` From 7957c5a92b19dde4ad440cc0720c60373fb3b46c Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:15:19 -0600 Subject: [PATCH 04/13] Add a state diff service --- cmd/geth/usage.go | 6 +++++ cmd/utils/flags.go | 13 +++++++++++ statediff/service.go | 54 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 statediff/service.go diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 82f17e0ee895..152ac059b0c0 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -245,6 +245,12 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerLegacyExtraDataFlag, }, }, + { + Name: "STATE DIFF", + Flags: []cli.Flag{ + utils.StateDiffFlag, + }, + }, { Name: "MISC", }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c18d98383fcb..b16f9cdcacd1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -58,6 +58,7 @@ import ( "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -1338,6 +1339,18 @@ func RegisterEthStatsService(stack *node.Node, url string) { } } +func RegisterStateDiffService(stack *node.Node) { + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + var ethServ *eth.Ethereum + ctx.Service(ðServ) + chainDb := ethServ.ChainDb() + blockChain := ethServ.BlockChain() + return statediff.NewStateDiffService(chainDb, blockChain) + }); err != nil { + Fatalf("Failed to register State Diff Service", err) + } +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/statediff/service.go b/statediff/service.go new file mode 100644 index 000000000000..695e3d638712 --- /dev/null +++ b/statediff/service.go @@ -0,0 +1,54 @@ +package statediff + +import ( + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/ethdb" + "fmt" +) + +type StateDiffService struct { + builder *builder + extractor *extractor + blockchain *core.BlockChain +} + +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { + config := Config{} + extractor, _ := NewExtractor(db, config) + return &StateDiffService{ + blockchain: blockChain, + extractor: extractor, + }, nil +} + +func (StateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} + +} + +func (StateDiffService) APIs() []rpc.API { + return []rpc.API{} +} + +func (sds *StateDiffService) Start(server *p2p.Server) error { + fmt.Println("starting the state diff service") + blockChannel := make(chan core.ChainHeadEvent) + sds.blockchain.SubscribeChainHeadEvent(blockChannel) + for { + select { + case <-blockChannel: + headOfChainEvent := <-blockChannel + previousBlock := headOfChainEvent.Block + //TODO: figure out the best way to get the previous block + currentBlock := headOfChainEvent.Block + sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) + } + } + return nil +} +func (StateDiffService) Stop() error { + fmt.Println("stopping the state diff service") + return nil +} \ No newline at end of file From 6fd20597a77fa0a1a5d0fd83d9c6bb82542589bc Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:20:25 -0600 Subject: [PATCH 05/13] Remove diff extractor from blockchain --- core/blockchain.go | 19 ------------------- eth/backend.go | 1 - 2 files changed, 20 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 77295b888cf2..1b48020a6c15 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -44,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" - "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -73,7 +72,6 @@ type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - StateDiff statediff.Config // Settings for state diff extraction } // BlockChain represents the canonical chain given a database with a genesis @@ -135,8 +133,6 @@ type BlockChain struct { badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. - - diffExtractor statediff.Extractor // State diff processing interface } // NewBlockChain returns a fully initialised block chain using information @@ -178,12 +174,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) var err error - if cacheConfig.StateDiff.On { - bc.diffExtractor, err = statediff.NewExtractor(db, cacheConfig.StateDiff) - if err != nil { - return nil, err - } - } bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { @@ -1216,15 +1206,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } proctime := time.Since(bstart) - // If extracting statediffs, do so now - if bc.cacheConfig.StateDiff.On { - // Currently not doing anything with returned cid... - _, err = bc.diffExtractor.ExtractStateDiff(*parent, *block) - if err != nil { - return i, events, coalescedLogs, err - } - } - // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) if err != nil { diff --git a/eth/backend.go b/eth/backend.go index d9617564351f..0de0a19803f2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -159,7 +159,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout, - StateDiff: config.StateDiff.On, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) From 32a0a238216db3fd988bf42f818cefb9d6fe5c9c Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 28 Nov 2018 16:51:16 -0600 Subject: [PATCH 06/13] Update imports --- statediff/ipfs/dag_putter.go | 3 ++- statediff/publisher.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/statediff/ipfs/dag_putter.go b/statediff/ipfs/dag_putter.go index 92ffa1e8aea6..4f99bdae37e2 100644 --- a/statediff/ipfs/dag_putter.go +++ b/statediff/ipfs/dag_putter.go @@ -22,9 +22,10 @@ package ipfs import ( "bytes" "encoding/gob" - "github.com/i-norden/go-ethereum/statediff" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/common" ) const ( diff --git a/statediff/publisher.go b/statediff/publisher.go index 163f8d3b0e4a..48ad7cf64550 100644 --- a/statediff/publisher.go +++ b/statediff/publisher.go @@ -21,7 +21,7 @@ package statediff import ( "errors" - "github.com/i-norden/go-ethereum/statediff/ipfs" + "github.com/ethereum/go-ethereum/statediff/ipfs" ) type Publisher interface { From 70d531dc1ef8418d1f03341891fbd06d20a7a93a Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 11:05:00 -0600 Subject: [PATCH 07/13] Move statediff on/off check to geth cmd config --- cmd/geth/config.go | 4 ++++ cmd/utils/flags.go | 11 ----------- eth/config.go | 4 ---- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index b0749d23291b..a05e78b112e6 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -178,6 +178,10 @@ func makeFullNode(ctx *cli.Context) *node.Node { if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, cfg.Ethstats.URL) } + + if ctx.GlobalBool(utils.StateDiffFlag.Name) { + utils.RegisterStateDiffService(stack) + } return stack } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b16f9cdcacd1..54e3dd83d1ba 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1134,13 +1134,6 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { } } -// Check if state diff flags are on and applies them to eth context -func setStateDiff(ctx *cli.Context, cfg *eth.Config) { - if ctx.GlobalBool(StateDiffFlag.Name) && cfg.NoPruning && cfg.SyncMode == downloader.FullSync { - cfg.StateDiff.On = true - } -} - // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags @@ -1176,10 +1169,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" - if ctx.GlobalIsSet(StateDiffFlag.Name) { - setStateDiff(ctx, cfg) - } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } diff --git a/eth/config.go b/eth/config.go index c675b6d08cf6..601f4735e33f 100644 --- a/eth/config.go +++ b/eth/config.go @@ -17,7 +17,6 @@ package eth import ( - "github.com/ethereum/go-ethereum/statediff" "math/big" "os" "os/user" @@ -129,9 +128,6 @@ type Config struct { EWASMInterpreter string // Type of the EVM interpreter ("" for default) EVMInterpreter string - - // Config for state diff building - StateDiff statediff.Config } type configMarshaling struct { From 71147861f70b808669f453550ee5bdf44db0610d Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 13:52:02 -0600 Subject: [PATCH 08/13] Update starting state diff service --- statediff/service.go | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/statediff/service.go b/statediff/service.go index 695e3d638712..6870ce04788d 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -1,11 +1,12 @@ package statediff import ( - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/ethdb" - "fmt" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/event" + "log" ) type StateDiffService struct { @@ -19,7 +20,7 @@ func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*State extractor, _ := NewExtractor(db, config) return &StateDiffService{ blockchain: blockChain, - extractor: extractor, + extractor: extractor, }, nil } @@ -32,23 +33,32 @@ func (StateDiffService) APIs() []rpc.API { return []rpc.API{} } -func (sds *StateDiffService) Start(server *p2p.Server) error { - fmt.Println("starting the state diff service") - blockChannel := make(chan core.ChainHeadEvent) - sds.blockchain.SubscribeChainHeadEvent(blockChannel) +func (sds *StateDiffService) loop (sub event.Subscription, events chan core.ChainHeadEvent) { + defer sub.Unsubscribe() + for { select { - case <-blockChannel: - headOfChainEvent := <-blockChannel - previousBlock := headOfChainEvent.Block + case ev, ok := <-events: + if !ok { + log.Fatalf("Error getting chain head event from subscription.") + } + log.Println("doing something with an event", ev) + previousBlock := ev.Block //TODO: figure out the best way to get the previous block - currentBlock := headOfChainEvent.Block + currentBlock := ev.Block sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) } } + +} +func (sds *StateDiffService) Start(server *p2p.Server) error { + events := make(chan core.ChainHeadEvent, 10) + sub := sds.blockchain.SubscribeChainHeadEvent(events) + + go sds.loop(sub, events) + return nil } func (StateDiffService) Stop() error { - fmt.Println("stopping the state diff service") return nil -} \ No newline at end of file +} From cf4ed9c2b378904a24cb42fe90d6af4e99a0aa66 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 29 Nov 2018 14:29:24 -0600 Subject: [PATCH 09/13] Add debugging logs for creating diff --- statediff/builder.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/statediff/builder.go b/statediff/builder.go index 433353591f5f..edfb50ca8740 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -49,10 +49,13 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block // Generate tries for old and new states oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) if err != nil { + log.Debug("error creating oldTrie", err) + //getting this error: error creating oldTrie missing trie node ddfbb83966d870891aa47147269447a83564d1defaefad5f9844a3a3a2a08433 (path ) return nil, err } newTrie, err := trie.New(newStateRoot, sdb.trieDB) if err != nil { + log.Debug("error creating newTrie", err) return nil, err } @@ -61,6 +64,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt := newTrie.NodeIterator([]byte{}) creations, err := sdb.collectDiffNodes(oldIt, newIt) if err != nil { + log.Debug("error collecting creation diff nodes", err) return nil, err } @@ -69,6 +73,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt = newTrie.NodeIterator(make([]byte, 0)) deletions, err := sdb.collectDiffNodes(newIt, oldIt) if err != nil { + log.Debug("error collecting deletion diff nodes", err) return nil, err } @@ -80,14 +85,17 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block // Build and return the statediff updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { + log.Debug("error building diff incremental for updated", err) return nil, err } createdAccounts, err := sdb.buildDiffEventual(creations, true) if err != nil { + log.Debug("error building diff incremental for created", err) return nil, err } deletedAccounts, err := sdb.buildDiffEventual(deletions, false) if err != nil { + log.Debug("error building diff incremental for deleted", err) return nil, err } From 097a9071ccee5df4f5f3b2a63936d89f945b6577 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 3 Dec 2018 16:23:15 -0600 Subject: [PATCH 10/13] Add state diff extractor test and small refactoring WIP - Add state diff builder test Clean up builder test --- statediff/builder.go | 34 ++-- statediff/builder_test.go | 299 ++++++++++++++++++++++++++++++ statediff/extractor.go | 20 +- statediff/extractor_test.go | 81 +++++++- statediff/service.go | 8 +- statediff/statediff_suite_test.go | 13 ++ statediff/struct.go | 22 +-- statediff/testhelpers/mocks.go | 49 +++++ 8 files changed, 483 insertions(+), 43 deletions(-) create mode 100644 statediff/statediff_suite_test.go create mode 100644 statediff/testhelpers/mocks.go diff --git a/statediff/builder.go b/statediff/builder.go index edfb50ca8740..da58b3bf4b50 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -109,7 +109,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block } func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { - var diffAccounts map[common.Address]*state.Account + var diffAccounts = make(map[common.Address]*state.Account) it, _ := trie.NewDifferenceIterator(a, b) for { @@ -161,15 +161,15 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account hexRoot := val.Root.Hex() if created { - nonce := diffUint64{ + nonce := DiffUint64{ NewValue: &val.Nonce, } - balance := diffBigInt{ + balance := DiffBigInt{ NewValue: val.Balance, } - contractRoot := diffString{ + contractRoot := DiffString{ NewValue: &hexRoot, } accountDiffs[addr] = AccountDiffEventual{ @@ -181,13 +181,13 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account Storage: storageDiffs, } } else { - nonce := diffUint64{ + nonce := DiffUint64{ OldValue: &val.Nonce, } - balance := diffBigInt{ + balance := DiffBigInt{ OldValue: val.Balance, } - contractRoot := diffString{ + contractRoot := DiffString{ OldValue: &hexRoot, } accountDiffs[addr] = AccountDiffEventual{ @@ -214,12 +214,12 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc log.Error("Failed building storage diffs", "Address", val, "error", err) return nil, err } else { - nonce := diffUint64{ + nonce := DiffUint64{ NewValue: &createdAcc.Nonce, OldValue: &deletedAcc.Nonce, } - balance := diffBigInt{ + balance := DiffBigInt{ NewValue: createdAcc.Balance, OldValue: deletedAcc.Balance, } @@ -227,7 +227,7 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc nHexRoot := createdAcc.Root.Hex() oHexRoot := deletedAcc.Root.Hex() - contractRoot := diffString{ + contractRoot := DiffString{ NewValue: &nHexRoot, OldValue: &oHexRoot, } @@ -246,14 +246,14 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc return updatedAccounts, nil } -func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]diffString, error) { +func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]DiffString, error) { log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) sTrie, err := trie.New(sr, sdb.trieDB) if err != nil { return nil, err } it := sTrie.NodeIterator(make([]byte, 0)) - storageDiffs := make(map[string]diffString) + storageDiffs := make(map[string]DiffString) for { log.Debug("Iterating over state at path ", "path", pathToStr(it)) if it.Leaf() { @@ -261,9 +261,9 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (ma path := pathToStr(it) value := common.ToHex(it.LeafBlob()) if creation { - storageDiffs[path] = diffString{NewValue: &value} + storageDiffs[path] = DiffString{NewValue: &value} } else { - storageDiffs[path] = diffString{OldValue: &value} + storageDiffs[path] = DiffString{OldValue: &value} } } cont := it.Next(true) @@ -274,7 +274,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (ma return storageDiffs, nil } -func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]diffString, error) { +func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]DiffString, error) { log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) oldTrie, err := trie.New(oldSR, sdb.trieDB) if err != nil { @@ -288,7 +288,7 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common oldIt := oldTrie.NodeIterator(make([]byte, 0)) newIt := newTrie.NodeIterator(make([]byte, 0)) it, _ := trie.NewDifferenceIterator(oldIt, newIt) - storageDiffs := make(map[string]diffString) + storageDiffs := make(map[string]DiffString) for { if it.Leaf() { log.Debug("Found leaf in storage", "path", pathToStr(it)) @@ -298,7 +298,7 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common log.Error("Failed to look up value in oldTrie", "path", path, "error", err) } else { hexOldVal := common.ToHex(oldVal) - storageDiffs[path] = diffString{OldValue: &hexOldVal, NewValue: &value} + storageDiffs[path] = DiffString{OldValue: &hexOldVal, NewValue: &value} } } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 6433b167a44d..64986af10a48 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -19,3 +19,302 @@ package statediff_test +import ( + "github.com/onsi/ginkgo" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "math/big" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/statediff" + "github.com/onsi/gomega" +) + + +var ( + testdb = ethdb.NewMemDatabase() + + testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + testBankFunds = big.NewInt(100000000) + genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) + + account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + + contractCode = common.Hex2Bytes("606060405260cc8060106000396000f360606040526000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146041578063c16431b914606b57603f565b005b6055600480803590602001909190505060a9565b6040518082815260200191505060405180910390f35b60886004808035906020019091908035906020019091905050608a565b005b80600060005083606481101560025790900160005b50819055505b5050565b6000600060005082606481101560025790900160005b5054905060c7565b91905056") + contractAddr common.Address + + emptyAccountDiffEventualMap = make(map[common.Address]statediff.AccountDiffEventual) + emptyAccountDiffIncrementalMap = make(map[common.Address]statediff.AccountDiffIncremental) +) +/* +contract test { + + uint256[100] data; + + function Put(uint256 addr, uint256 value) { + data[addr] = value; + } + + function Get(uint256 addr) constant returns (uint256 value) { + return data[addr]; + } +} +*/ + + +// makeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 3rd block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { + blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, testChainGen) + hashes := make([]common.Hash, n+1) + hashes[len(hashes)-1] = parent.Hash() + blockm := make(map[common.Hash]*types.Block, n+1) + blockm[parent.Hash()] = parent + for i, b := range blocks { + hashes[len(hashes)-i-2] = b.Hash() + blockm[b.Hash()] = b + } + return hashes, blockm +} + + +func testChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // account1Addr passes it on to account #2. + // account1Addr creates a test contract. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + nonce := block.TxNonce(account1Addr) + tx2, _ := types.SignTx(types.NewTransaction(nonce, account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, account1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), contractCode), signer, account1Key) + contractAddr = crypto.CreateAddress(account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592 + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(account2Addr) + //get function: 60cd2685 + //put function: c16431b9 + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), contractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) + block.AddTx(tx) + } +} +var _ = ginkgo.FDescribe("", func() { + var ( + block0Hash, block1Hash, block2Hash, block3Hash common.Hash + block0, block1, block2, block3 *types.Block + builder statediff.Builder + miningReward = int64(3000000000000000000) + burnAddress = common.HexToAddress("0x0") + diff *statediff.StateDiff + err error + ) + + ginkgo.BeforeEach(func() { + _, blocks := makeChain(3, 0, genesis) + block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash = common.HexToHash("0x47c398dd688eaa4dd11b006888156783fe32df83d59b197c0fcd303408103d39") + block2Hash = common.HexToHash("0x351b2f531838683ba457e8ca4d3a844cc48147dceafbcb589dc6e3227856ee75") + block3Hash = common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73") + + block0 = blocks[block0Hash] + block1 = blocks[block1Hash] + block2 = blocks[block2Hash] + block3 = blocks[block3Hash] + builder = statediff.NewBuilder(testdb) + }) + + ginkgo.It("returns empty account diff collections when the state root hasn't changed", func() { + expectedDiff := statediff.StateDiff{ + BlockNumber: block0.Number().Int64(), + BlockHash: block0Hash, + CreatedAccounts: emptyAccountDiffEventualMap, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: emptyAccountDiffIncrementalMap, + } + + diff, err := builder.BuildStateDiff(block0.Root(), block0.Root(), block0.Number().Int64(), block0Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(diff).To(gomega.Equal(&expectedDiff)) + }) + + ginkgo.Context("Block 1", func() { + //10000 transferred from testBankAddress to account1Addr + var balanceChange = int64(10000) + + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block0.Root(), block1.Root(), block1.Number().Int64(), block1Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block1.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block1Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance diffs for updated accounts", func() { + expectedBankBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(testBankFunds.Int64() - balanceChange), + OldValue: testBankFunds, + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(1)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Balance).To(gomega.Equal(expectedBankBalanceDiff)) + }) + + ginkgo.It("returns balance diffs for new accounts", func() { + expectedAccount1BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(balanceChange), + OldValue: nil, + } + + expectedBurnAddrBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(miningReward), + OldValue: nil, + } + + gomega.Expect(len(diff.CreatedAccounts)).To(gomega.Equal(2)) + gomega.Expect(diff.CreatedAccounts[account1Addr].Balance).To(gomega.Equal(expectedAccount1BalanceDiff)) + gomega.Expect(diff.CreatedAccounts[burnAddress].Balance).To(gomega.Equal(expectedBurnAddrBalanceDiff)) + }) + }) + + ginkgo.Context("Block 2", func() { + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + var ( + balanceChange = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + ) + + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block1.Root(), block2.Root(), block2.Number().Int64(), block2Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block2.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block2Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance diffs for updated accounts", func() { + expectedBankBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block1BankBalance - balanceChange), + OldValue: big.NewInt(block1BankBalance), + } + + expectedAccount1BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block1Account1Balance - balanceChange + balanceChange), + OldValue: big.NewInt(block1Account1Balance), + } + + expectedBurnBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(miningReward + miningReward), + OldValue: big.NewInt(miningReward), + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(3)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Balance).To(gomega.Equal(expectedBankBalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[account1Addr].Balance).To(gomega.Equal(expectedAccount1BalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[burnAddress].Balance).To(gomega.Equal(expectedBurnBalanceDiff)) + }) + + ginkgo.It("returns balance diffs for new accounts", func() { + expectedAccount2BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(balanceChange), + OldValue: nil, + } + + expectedContractBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(0), + OldValue: nil, + } + + gomega.Expect(len(diff.CreatedAccounts)).To(gomega.Equal(2)) + gomega.Expect(diff.CreatedAccounts[account2Addr].Balance).To(gomega.Equal(expectedAccount2BalanceDiff)) + gomega.Expect(diff.CreatedAccounts[contractAddr].Balance).To(gomega.Equal(expectedContractBalanceDiff)) + }) + }) + + ginkgo.Context("Block 3", func() { + //the contract's storage is changed + //and the block is mined by account 2 + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block2.Root(), block3.Root(), block3.Number().Int64(), block3Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block3.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block3Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns an empty collection for created accounts", func() { + gomega.Expect(diff.CreatedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance, storage and nonce diffs for updated accounts", func() { + block2Account2Balance := int64(1000) + expectedAcct2BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block2Account2Balance + miningReward), + OldValue: big.NewInt(block2Account2Balance), + } + + expectedContractStorageDiff := make(map[string]statediff.DiffString) + newVal := "0x03" + oldVal := "0x0" + path := "0x405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace" + expectedContractStorageDiff[path] = statediff.DiffString{ + NewValue: &newVal, + OldValue: &oldVal, + } + + oldNonce := uint64(2) + newNonce := uint64(3) + expectedBankNonceDiff := statediff.DiffUint64{ + NewValue: &newNonce, + OldValue: &oldNonce, + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(3)) + gomega.Expect(diff.UpdatedAccounts[account2Addr].Balance).To(gomega.Equal(expectedAcct2BalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[contractAddr].Storage).To(gomega.Equal(expectedContractStorageDiff)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Nonce).To(gomega.Equal(expectedBankNonceDiff)) + }) + }) +}) \ No newline at end of file diff --git a/statediff/extractor.go b/statediff/extractor.go index 3cbb5a311604..9d9b1f080c37 100644 --- a/statediff/extractor.go +++ b/statediff/extractor.go @@ -21,7 +21,6 @@ package statediff import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" ) type Extractor interface { @@ -29,27 +28,22 @@ type Extractor interface { } type extractor struct { - *builder // Interface for building state diff objects from two blocks - *publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) + Builder Builder // Interface for building state diff objects from two blocks + Publisher Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) } -func NewExtractor(db ethdb.Database, config Config) (*extractor, error) { - publisher, err := NewPublisher(config) - if err != nil { - return nil, err - } - +func NewExtractor(builder Builder, publisher Publisher) (*extractor, error) { return &extractor{ - builder: NewBuilder(db), - publisher: publisher, + Builder: builder, + Publisher: publisher, }, nil } func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) { - stateDiff, err := e.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) + stateDiff, err := e.Builder.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) if err != nil { return "", err } - return e.PublishStateDiff(stateDiff) + return e.Publisher.PublishStateDiff(stateDiff) } \ No newline at end of file diff --git a/statediff/extractor_test.go b/statediff/extractor_test.go index de5e170957b0..9ccd250f6c95 100644 --- a/statediff/extractor_test.go +++ b/statediff/extractor_test.go @@ -17,4 +17,83 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff_test \ No newline at end of file +package statediff_test + +import ( + "github.com/onsi/ginkgo" + "github.com/ethereum/go-ethereum/statediff" + "github.com/onsi/gomega" + "github.com/ethereum/go-ethereum/core/types" + "math/rand" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + "math/big" +) +var _ = ginkgo.Describe("Extractor", func() { + var publisher testhelpers.MockPublisher + var builder testhelpers.MockBuilder + var currentBlockNumber *big.Int + var parentBlock, currentBlock *types.Block + var expectedStateDiff statediff.StateDiff + var extractor statediff.Extractor + var err error + + ginkgo.BeforeEach(func() { + publisher = testhelpers.MockPublisher{} + builder = testhelpers.MockBuilder{} + extractor, err = statediff.NewExtractor(&builder, &publisher) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + blockNumber := rand.Int63() + parentBlockNumber := big.NewInt(blockNumber - int64(1)) + currentBlockNumber = big.NewInt(blockNumber) + parentBlock = types.NewBlock(&types.Header{Number: parentBlockNumber}, nil, nil, nil) + currentBlock = types.NewBlock(&types.Header{Number: currentBlockNumber}, nil, nil, nil) + + expectedStateDiff = statediff.StateDiff{ + BlockNumber: blockNumber, + BlockHash: currentBlock.Hash(), + CreatedAccounts: nil, + DeletedAccounts: nil, + UpdatedAccounts: nil, + } + }) + + ginkgo.It("builds a state diff struct", func() { + builder.SetStateDiffToBuild(&expectedStateDiff) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(builder.OldStateRoot).To(gomega.Equal(parentBlock.Root())) + gomega.Expect(builder.NewStateRoot).To(gomega.Equal(currentBlock.Root())) + gomega.Expect(builder.BlockNumber).To(gomega.Equal(currentBlockNumber.Int64())) + gomega.Expect(builder.BlockHash).To(gomega.Equal(currentBlock.Hash())) + }) + + ginkgo.It("returns an error if building the state diff fails", func() { + builder.SetBuilderError(testhelpers.MockError) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err).To(gomega.MatchError(testhelpers.MockError)) + }) + + ginkgo.It("publishes the state diff struct", func() { + builder.SetStateDiffToBuild(&expectedStateDiff) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(publisher.StateDiff).To(gomega.Equal(&expectedStateDiff)) + }) + + ginkgo.It("returns an error if publishing the diff fails", func() { + publisher.SetPublisherError(testhelpers.MockError) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err).To(gomega.MatchError(testhelpers.MockError)) + }) +}) diff --git a/statediff/service.go b/statediff/service.go index 6870ce04788d..86839cf3b83b 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -17,7 +17,13 @@ type StateDiffService struct { func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { config := Config{} - extractor, _ := NewExtractor(db, config) + builder := NewBuilder(db) + publisher, err := NewPublisher(config) + if err != nil { + return nil, nil + } + + extractor, _ := NewExtractor(builder, publisher) return &StateDiffService{ blockchain: blockChain, extractor: extractor, diff --git a/statediff/statediff_suite_test.go b/statediff/statediff_suite_test.go new file mode 100644 index 000000000000..7118ea792291 --- /dev/null +++ b/statediff/statediff_suite_test.go @@ -0,0 +1,13 @@ +package statediff_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStatediff(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Statediff Suite") +} diff --git a/statediff/struct.go b/statediff/struct.go index 2142ec755c90..009af4c2ea6c 100644 --- a/statediff/struct.go +++ b/statediff/struct.go @@ -56,31 +56,31 @@ func (sd *StateDiff) Encode() ([]byte, error) { } type AccountDiffEventual struct { - Nonce diffUint64 `json:"nonce" gencodec:"required"` - Balance diffBigInt `json:"balance" gencodec:"required"` + Nonce DiffUint64 `json:"nonce" gencodec:"required"` + Balance DiffBigInt `json:"balance" gencodec:"required"` Code []byte `json:"code" gencodec:"required"` CodeHash string `json:"codeHash" gencodec:"required"` - ContractRoot diffString `json:"contractRoot" gencodec:"required"` - Storage map[string]diffString `json:"storage" gencodec:"required"` + ContractRoot DiffString `json:"contractRoot" gencodec:"required"` + Storage map[string]DiffString `json:"storage" gencodec:"required"` } type AccountDiffIncremental struct { - Nonce diffUint64 `json:"nonce" gencodec:"required"` - Balance diffBigInt `json:"balance" gencodec:"required"` + Nonce DiffUint64 `json:"nonce" gencodec:"required"` + Balance DiffBigInt `json:"balance" gencodec:"required"` CodeHash string `json:"codeHash" gencodec:"required"` - ContractRoot diffString `json:"contractRoot" gencodec:"required"` - Storage map[string]diffString `json:"storage" gencodec:"required"` + ContractRoot DiffString `json:"contractRoot" gencodec:"required"` + Storage map[string]DiffString `json:"storage" gencodec:"required"` } -type diffString struct { +type DiffString struct { NewValue *string `json:"newValue" gencodec:"optional"` OldValue *string `json:"oldValue" gencodec:"optional"` } -type diffUint64 struct { +type DiffUint64 struct { NewValue *uint64 `json:"newValue" gencodec:"optional"` OldValue *uint64 `json:"oldValue" gencodec:"optional"` } -type diffBigInt struct { +type DiffBigInt struct { NewValue *big.Int `json:"newValue" gencodec:"optional"` OldValue *big.Int `json:"oldValue" gencodec:"optional"` } diff --git a/statediff/testhelpers/mocks.go b/statediff/testhelpers/mocks.go new file mode 100644 index 000000000000..4bff3c02862d --- /dev/null +++ b/statediff/testhelpers/mocks.go @@ -0,0 +1,49 @@ +package testhelpers + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff" + "errors" +) + +var MockError = errors.New("mock error") + +type MockBuilder struct { + OldStateRoot common.Hash + NewStateRoot common.Hash + BlockNumber int64 + BlockHash common.Hash + stateDiff *statediff.StateDiff + builderError error +} + +func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*statediff.StateDiff, error) { + builder.OldStateRoot = oldStateRoot + builder.NewStateRoot = newStateRoot + builder.BlockNumber = blockNumber + builder.BlockHash = blockHash + + return builder.stateDiff, builder.builderError +} + +func (builder *MockBuilder) SetStateDiffToBuild(stateDiff *statediff.StateDiff) { + builder.stateDiff = stateDiff +} + +func (builder *MockBuilder) SetBuilderError(err error) { + builder.builderError = err +} + +type MockPublisher struct{ + StateDiff *statediff.StateDiff + publisherError error +} + +func (publisher *MockPublisher) PublishStateDiff(sd *statediff.StateDiff) (string, error) { + publisher.StateDiff = sd + return "", publisher.publisherError +} + +func (publisher *MockPublisher) SetPublisherError(err error) { + publisher.publisherError = err +} From 45e8f0719621504ff700c44ddc4200f7dbacef62 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 12 Dec 2018 16:18:24 -0600 Subject: [PATCH 11/13] WIP - writing state diff to a CSV --- statediff/config.go | 9 +- statediff/publisher.go | 160 ++++++++++++++++++++++--- statediff/publisher_test.go | 226 ++++++++++++++++++++++++++++++++---- 3 files changed, 355 insertions(+), 40 deletions(-) diff --git a/statediff/config.go b/statediff/config.go index ac207acd1347..fbc258be2b27 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -30,7 +30,8 @@ type Config struct { type StateDiffMode int const ( - IPLD StateDiffMode = iota + CSV StateDiffMode = iota + IPLD LDB SQL ) @@ -42,6 +43,8 @@ func (mode StateDiffMode) IsValid() bool { // String implements the stringer interface. func (mode StateDiffMode) String() string { switch mode { + case CSV: + return "csv" case IPLD: return "ipfs" case LDB: @@ -55,6 +58,8 @@ func (mode StateDiffMode) String() string { func (mode StateDiffMode) MarshalText() ([]byte, error) { switch mode { + case CSV: + return []byte("ipfs"), nil case IPLD: return []byte("ipfs"), nil case LDB: @@ -68,6 +73,8 @@ func (mode StateDiffMode) MarshalText() ([]byte, error) { func (mode *StateDiffMode) UnmarshalText(text []byte) error { switch string(text) { + case "csv": + *mode = CSV case "ipfs": *mode = IPLD case "ldb": diff --git a/statediff/publisher.go b/statediff/publisher.go index 48ad7cf64550..e6526cee9d7f 100644 --- a/statediff/publisher.go +++ b/statediff/publisher.go @@ -20,8 +20,11 @@ package statediff import ( - "errors" - "github.com/ethereum/go-ethereum/statediff/ipfs" + "os" + "encoding/csv" + "time" + "strconv" + "strings" ) type Publisher interface { @@ -29,35 +32,154 @@ type Publisher interface { } type publisher struct { - ipfs.DagPutter - Config + Config Config } -func NewPublisher(config Config) (*publisher, error) { - adder, err := ipfs.NewAdder(config.Path) - if err != nil { - return nil, err +var ( + Headers = []string{ + "blockNumber", "blockHash", "accountAction", + "code", "codeHash", + "oldNonceValue", "newNonceValue", + "oldBalanceValue", "newBalanceValue", + "oldContractRoot", "newContractRoot", + "storageDiffPaths", } + timeStampFormat = "20060102150405.00000" + deletedAccountAction = "deleted" + createdAccountAction = "created" + updatedAccountAction = "updated" +) + +func NewPublisher(config Config) (*publisher, error) { return &publisher{ - DagPutter: ipfs.NewDagPutter(adder), Config: config, }, nil } func (p *publisher) PublishStateDiff(sd *StateDiff) (string, error) { - switch p.Mode { - case IPLD: - cidStr, err := p.DagPut(sd) + switch p.Config.Mode { + case CSV: + return "", p.publishStateDiffToCSV(*sd) + default: + return "", p.publishStateDiffToCSV(*sd) + } +} + +func (p *publisher) publishStateDiffToCSV(sd StateDiff) error { + now := time.Now() + timeStamp := now.Format(timeStampFormat) + filePath := p.Config.Path + timeStamp + ".csv" + file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + writer := csv.NewWriter(file) + defer writer.Flush() + + var data [][]string + data = append(data, Headers) + for _, row := range accumulateCreatedAccountRows(sd) { + data = append(data, row) + } + for _, row := range accumulateUpdatedAccountRows(sd) { + data = append(data, row) + } + + for _, row := range accumulateDeletedAccountRows(sd) { + data = append(data, row) + } + + for _, value := range data{ + err := writer.Write(value) if err != nil { - return "", err + return err } + } - return cidStr, err - case LDB: - case SQL: - default: + return nil +} + +func accumulateUpdatedAccountRows(sd StateDiff) [][]string { + var updatedAccountRows [][]string + for _, accountDiff := range sd.UpdatedAccounts { + formattedAccountData := formatAccountDiffIncremental(accountDiff, sd, updatedAccountAction) + + updatedAccountRows = append(updatedAccountRows, formattedAccountData) + } + + return updatedAccountRows +} + +func accumulateDeletedAccountRows(sd StateDiff) [][]string { + var deletedAccountRows [][]string + for _, accountDiff := range sd.DeletedAccounts { + formattedAccountData := formatAccountDiffEventual(accountDiff, sd, deletedAccountAction) + + deletedAccountRows = append(deletedAccountRows, formattedAccountData) + } + + return deletedAccountRows +} + +func accumulateCreatedAccountRows(sd StateDiff) [][]string { + var createdAccountRows [][]string + for _, accountDiff := range sd.CreatedAccounts { + formattedAccountData := formatAccountDiffEventual(accountDiff, sd, createdAccountAction) + + createdAccountRows = append(createdAccountRows, formattedAccountData) } - return "", errors.New("state diff publisher: unhandled publishing mode") -} \ No newline at end of file + return createdAccountRows +} + +func formatAccountDiffEventual(accountDiff AccountDiffEventual, sd StateDiff, accountAction string) []string { + oldContractRoot := accountDiff.ContractRoot.OldValue + newContractRoot := accountDiff.ContractRoot.NewValue + var storageDiffPaths []string + for k := range accountDiff.Storage { + storageDiffPaths = append(storageDiffPaths, k) + } + formattedAccountData := []string{ + strconv.FormatInt(sd.BlockNumber, 10), + sd.BlockHash.String(), + accountAction, + string(accountDiff.Code), + accountDiff.CodeHash, + strconv.FormatUint(*accountDiff.Nonce.OldValue, 10), + strconv.FormatUint(*accountDiff.Nonce.NewValue, 10), + accountDiff.Balance.OldValue.String(), + accountDiff.Balance.NewValue.String(), + *oldContractRoot, + *newContractRoot, + strings.Join(storageDiffPaths, ","), + } + return formattedAccountData +} + +func formatAccountDiffIncremental(accountDiff AccountDiffIncremental, sd StateDiff, accountAction string) []string { + oldContractRoot := accountDiff.ContractRoot.OldValue + newContractRoot := accountDiff.ContractRoot.NewValue + var storageDiffPaths []string + for k := range accountDiff.Storage { + storageDiffPaths = append(storageDiffPaths, k) + } + formattedAccountData := []string{ + strconv.FormatInt(sd.BlockNumber, 10), + sd.BlockHash.String(), + accountAction, + "", + accountDiff.CodeHash, + strconv.FormatUint(*accountDiff.Nonce.OldValue, 10), + strconv.FormatUint(*accountDiff.Nonce.NewValue, 10), + accountDiff.Balance.OldValue.String(), + accountDiff.Balance.NewValue.String(), + *oldContractRoot, + *newContractRoot, + strings.Join(storageDiffPaths, ","), + } + return formattedAccountData +} + diff --git a/statediff/publisher_test.go b/statediff/publisher_test.go index de5e170957b0..9d34c210749e 100644 --- a/statediff/publisher_test.go +++ b/statediff/publisher_test.go @@ -1,20 +1,206 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package statediff_test \ No newline at end of file +package statediff_test + +import ( + "github.com/onsi/ginkgo" + "github.com/ethereum/go-ethereum/statediff" + "github.com/onsi/gomega" + "os" + "encoding/csv" + "github.com/ethereum/go-ethereum/common" + "math/rand" + "math/big" + "path/filepath" + "strings" + "strconv" +) + +var _ = ginkgo.Describe("Publisher", func() { + ginkgo.Context("default CSV publisher", func() { + var ( + publisher statediff.Publisher + err error + config = statediff.Config{ + Path: "./test-", + } + ) + + var ( + blockNumber = rand.Int63() + blockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" + codeHash = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + oldNonceValue = rand.Uint64() + newNonceValue = oldNonceValue + 1 + oldBalanceValue = rand.Int63() + newBalanceValue = oldBalanceValue - 1 + contractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + storagePath = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + oldStorage = "0x0" + newStorage = "0x03" + storage = map[string]statediff.DiffString{storagePath: { + NewValue: &newStorage, + OldValue: &oldStorage, + }} + address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + createdAccounts = map[common.Address]statediff.AccountDiffEventual{address: { + Nonce: statediff.DiffUint64{ + NewValue: &newNonceValue, + OldValue: &oldNonceValue, + }, + Balance: statediff.DiffBigInt{ + NewValue: big.NewInt(newBalanceValue), + OldValue: big.NewInt(oldBalanceValue), + }, + ContractRoot: statediff.DiffString{ + NewValue: &contractRoot, + OldValue: &contractRoot, + }, + Code: []byte("created account code"), + CodeHash: codeHash, + Storage: storage, + }} + + updatedAccounts = map[common.Address]statediff.AccountDiffIncremental{address: { + Nonce: statediff.DiffUint64{ + NewValue: &newNonceValue, + OldValue: &oldNonceValue, + }, + Balance: statediff.DiffBigInt{ + NewValue: big.NewInt(newBalanceValue), + OldValue: big.NewInt(oldBalanceValue), + }, + CodeHash: codeHash, + ContractRoot: statediff.DiffString{ + NewValue: &contractRoot, + OldValue: &contractRoot, + }, + Storage: storage, + }} + + deletedAccounts = map[common.Address]statediff.AccountDiffEventual{address: { + Nonce: statediff.DiffUint64{ + NewValue: &newNonceValue, + OldValue: &oldNonceValue, + }, + Balance: statediff.DiffBigInt{ + NewValue: big.NewInt(newBalanceValue), + OldValue: big.NewInt(oldBalanceValue), + }, + ContractRoot: statediff.DiffString{ + NewValue: &contractRoot, + OldValue: &contractRoot, + }, + Code: []byte("deleted account code"), + CodeHash: codeHash, + Storage: storage, + }} + + testStateDiff = statediff.StateDiff{ + BlockNumber: blockNumber, + BlockHash: common.HexToHash(blockHash), + CreatedAccounts: createdAccounts, + DeletedAccounts: deletedAccounts, + UpdatedAccounts: updatedAccounts, + } + ) + + var lines [][]string + var file *os.File + ginkgo.BeforeEach(func() { + publisher, err = statediff.NewPublisher(config) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + _, err := publisher.PublishStateDiff(&testStateDiff) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + filePaths := getTestCSVFiles(".") + file, err = os.Open(filePaths[0]) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + defer file.Close() + + lines, err = csv.NewReader(file).ReadAll() + }) + + ginkgo.AfterEach(func() { + os.Remove(file.Name()) + }) + + ginkgo.It("persists the column headers to a CSV file", func() { + gomega.Expect(len(lines) > 1).To(gomega.BeTrue()) + gomega.Expect(lines[0]).To(gomega.Equal(statediff.Headers)) + }) + + ginkgo.It("persists the created account diffs to a CSV file", func() { + expectedCreatedAccountRow := []string{ + strconv.FormatInt(blockNumber, 10), + blockHash, + "created", + "created account code", + codeHash, + strconv.FormatUint(oldNonceValue, 10), + strconv.FormatUint(newNonceValue, 10), + strconv.FormatInt(oldBalanceValue, 10), + strconv.FormatInt(newBalanceValue, 10), + contractRoot, + contractRoot, + storagePath, + } + + gomega.Expect(len(lines) > 1).To(gomega.BeTrue()) + gomega.Expect(lines[1]).To(gomega.Equal(expectedCreatedAccountRow)) + }) + + ginkgo.It("persists the updated account diffs to a CSV file", func() { + expectedUpdatedAccountRow := []string{ + strconv.FormatInt(blockNumber, 10), + blockHash, + "updated", + "", + codeHash, + strconv.FormatUint(oldNonceValue, 10), + strconv.FormatUint(newNonceValue, 10), + strconv.FormatInt(oldBalanceValue, 10), + strconv.FormatInt(newBalanceValue, 10), + contractRoot, + contractRoot, + storagePath, + } + + gomega.Expect(len(lines) > 2).To(gomega.BeTrue()) + gomega.Expect(lines[2]).To(gomega.Equal(expectedUpdatedAccountRow)) + }) + + ginkgo.It("persists the deleted account diffs to a CSV file", func() { + expectedDeletedAccountRow := []string{ + strconv.FormatInt(blockNumber, 10), + blockHash, + "deleted", + "deleted account code", + codeHash, + strconv.FormatUint(oldNonceValue, 10), + strconv.FormatUint(newNonceValue, 10), + strconv.FormatInt(oldBalanceValue, 10), + strconv.FormatInt(newBalanceValue, 10), + contractRoot, + contractRoot, + storagePath, + } + + gomega.Expect(len(lines) > 3).To(gomega.BeTrue()) + gomega.Expect(lines[3]).To(gomega.Equal(expectedDeletedAccountRow)) + }) + }) +}) + +func getTestCSVFiles(rootPath string) []string{ + var files []string + err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if strings.HasPrefix(path, "test-") { + files = append(files, path) + } + return nil + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return files +} + From 3a2600a7184d34bbe81423ed61b35160e129ffb8 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 13 Dec 2018 10:59:31 -0600 Subject: [PATCH 12/13] Restructure statediff directory --- cmd/utils/flags.go | 4 +- statediff/{ => builder}/builder.go | 3 +- statediff/builder/builder_suite_test.go | 13 ++++ statediff/{ => builder}/builder_test.go | 40 +++++------ statediff/{ => builder}/helpers.go | 2 +- statediff/{ => builder}/struct.go | 2 +- statediff/{ => extractor}/extractor.go | 10 +-- statediff/extractor/extractor_suite_test.go | 13 ++++ statediff/{ => extractor}/extractor_test.go | 13 ++-- statediff/ipfs/adder.go | 58 --------------- statediff/ipfs/dag_putter.go | 80 --------------------- statediff/ipfs/helpers.go | 38 ---------- statediff/ipfs/node.go | 78 -------------------- statediff/{ => publisher}/publisher.go | 26 +++---- statediff/publisher/publisher_suite_test.go | 13 ++++ statediff/{ => publisher}/publisher_test.go | 39 +++++----- statediff/{ => service}/service.go | 18 +++-- statediff/testhelpers/mocks.go | 12 ++-- 18 files changed, 128 insertions(+), 334 deletions(-) rename statediff/{ => builder}/builder.go (98%) create mode 100644 statediff/builder/builder_suite_test.go rename statediff/{ => builder}/builder_test.go (91%) rename statediff/{ => builder}/helpers.go (99%) rename statediff/{ => builder}/struct.go (99%) rename statediff/{ => extractor}/extractor.go (78%) create mode 100644 statediff/extractor/extractor_suite_test.go rename statediff/{ => extractor}/extractor_test.go (92%) delete mode 100644 statediff/ipfs/adder.go delete mode 100644 statediff/ipfs/dag_putter.go delete mode 100644 statediff/ipfs/helpers.go delete mode 100644 statediff/ipfs/node.go rename statediff/{ => publisher}/publisher.go (83%) create mode 100644 statediff/publisher/publisher_suite_test.go rename statediff/{ => publisher}/publisher_test.go (84%) rename statediff/{ => service}/service.go (76%) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 54e3dd83d1ba..1c9b44d531ac 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -58,7 +58,7 @@ import ( "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "gopkg.in/urfave/cli.v1" - "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/service" ) var ( @@ -1334,7 +1334,7 @@ func RegisterStateDiffService(stack *node.Node) { ctx.Service(ðServ) chainDb := ethServ.ChainDb() blockChain := ethServ.BlockChain() - return statediff.NewStateDiffService(chainDb, blockChain) + return service.NewStateDiffService(chainDb, blockChain) }); err != nil { Fatalf("Failed to register State Diff Service", err) } diff --git a/statediff/builder.go b/statediff/builder/builder.go similarity index 98% rename from statediff/builder.go rename to statediff/builder/builder.go index da58b3bf4b50..404c98ad1a5c 100644 --- a/statediff/builder.go +++ b/statediff/builder/builder.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package builder import ( "github.com/ethereum/go-ethereum/common" @@ -50,7 +50,6 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) if err != nil { log.Debug("error creating oldTrie", err) - //getting this error: error creating oldTrie missing trie node ddfbb83966d870891aa47147269447a83564d1defaefad5f9844a3a3a2a08433 (path ) return nil, err } newTrie, err := trie.New(newStateRoot, sdb.trieDB) diff --git a/statediff/builder/builder_suite_test.go b/statediff/builder/builder_suite_test.go new file mode 100644 index 000000000000..2cfa71d5a7eb --- /dev/null +++ b/statediff/builder/builder_suite_test.go @@ -0,0 +1,13 @@ +package builder_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestBuilder(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Builder Suite") +} diff --git a/statediff/builder_test.go b/statediff/builder/builder_test.go similarity index 91% rename from statediff/builder_test.go rename to statediff/builder/builder_test.go index 64986af10a48..5fc4fca13642 100644 --- a/statediff/builder_test.go +++ b/statediff/builder/builder_test.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff_test +package builder_test import ( "github.com/onsi/ginkgo" @@ -29,8 +29,8 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "math/big" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/statediff" "github.com/onsi/gomega" + b "github.com/ethereum/go-ethereum/statediff/builder" ) @@ -50,8 +50,8 @@ var ( contractCode = common.Hex2Bytes("606060405260cc8060106000396000f360606040526000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146041578063c16431b914606b57603f565b005b6055600480803590602001909190505060a9565b6040518082815260200191505060405180910390f35b60886004808035906020019091908035906020019091905050608a565b005b80600060005083606481101560025790900160005b50819055505b5050565b6000600060005082606481101560025790900160005b5054905060c7565b91905056") contractAddr common.Address - emptyAccountDiffEventualMap = make(map[common.Address]statediff.AccountDiffEventual) - emptyAccountDiffIncrementalMap = make(map[common.Address]statediff.AccountDiffIncremental) + emptyAccountDiffEventualMap = make(map[common.Address]b.AccountDiffEventual) + emptyAccountDiffIncrementalMap = make(map[common.Address]b.AccountDiffIncremental) ) /* contract test { @@ -121,10 +121,10 @@ var _ = ginkgo.FDescribe("", func() { var ( block0Hash, block1Hash, block2Hash, block3Hash common.Hash block0, block1, block2, block3 *types.Block - builder statediff.Builder + builder b.Builder miningReward = int64(3000000000000000000) burnAddress = common.HexToAddress("0x0") - diff *statediff.StateDiff + diff *b.StateDiff err error ) @@ -139,11 +139,11 @@ var _ = ginkgo.FDescribe("", func() { block1 = blocks[block1Hash] block2 = blocks[block2Hash] block3 = blocks[block3Hash] - builder = statediff.NewBuilder(testdb) + builder = b.NewBuilder(testdb) }) ginkgo.It("returns empty account diff collections when the state root hasn't changed", func() { - expectedDiff := statediff.StateDiff{ + expectedDiff := b.StateDiff{ BlockNumber: block0.Number().Int64(), BlockHash: block0Hash, CreatedAccounts: emptyAccountDiffEventualMap, @@ -177,7 +177,7 @@ var _ = ginkgo.FDescribe("", func() { }) ginkgo.It("returns balance diffs for updated accounts", func() { - expectedBankBalanceDiff := statediff.DiffBigInt{ + expectedBankBalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(testBankFunds.Int64() - balanceChange), OldValue: testBankFunds, } @@ -187,12 +187,12 @@ var _ = ginkgo.FDescribe("", func() { }) ginkgo.It("returns balance diffs for new accounts", func() { - expectedAccount1BalanceDiff := statediff.DiffBigInt{ + expectedAccount1BalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(balanceChange), OldValue: nil, } - expectedBurnAddrBalanceDiff := statediff.DiffBigInt{ + expectedBurnAddrBalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(miningReward), OldValue: nil, } @@ -228,17 +228,17 @@ var _ = ginkgo.FDescribe("", func() { }) ginkgo.It("returns balance diffs for updated accounts", func() { - expectedBankBalanceDiff := statediff.DiffBigInt{ + expectedBankBalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(block1BankBalance - balanceChange), OldValue: big.NewInt(block1BankBalance), } - expectedAccount1BalanceDiff := statediff.DiffBigInt{ + expectedAccount1BalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(block1Account1Balance - balanceChange + balanceChange), OldValue: big.NewInt(block1Account1Balance), } - expectedBurnBalanceDiff := statediff.DiffBigInt{ + expectedBurnBalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(miningReward + miningReward), OldValue: big.NewInt(miningReward), } @@ -250,12 +250,12 @@ var _ = ginkgo.FDescribe("", func() { }) ginkgo.It("returns balance diffs for new accounts", func() { - expectedAccount2BalanceDiff := statediff.DiffBigInt{ + expectedAccount2BalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(balanceChange), OldValue: nil, } - expectedContractBalanceDiff := statediff.DiffBigInt{ + expectedContractBalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(0), OldValue: nil, } @@ -290,23 +290,23 @@ var _ = ginkgo.FDescribe("", func() { ginkgo.It("returns balance, storage and nonce diffs for updated accounts", func() { block2Account2Balance := int64(1000) - expectedAcct2BalanceDiff := statediff.DiffBigInt{ + expectedAcct2BalanceDiff := b.DiffBigInt{ NewValue: big.NewInt(block2Account2Balance + miningReward), OldValue: big.NewInt(block2Account2Balance), } - expectedContractStorageDiff := make(map[string]statediff.DiffString) + expectedContractStorageDiff := make(map[string]b.DiffString) newVal := "0x03" oldVal := "0x0" path := "0x405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace" - expectedContractStorageDiff[path] = statediff.DiffString{ + expectedContractStorageDiff[path] = b.DiffString{ NewValue: &newVal, OldValue: &oldVal, } oldNonce := uint64(2) newNonce := uint64(3) - expectedBankNonceDiff := statediff.DiffUint64{ + expectedBankNonceDiff := b.DiffUint64{ NewValue: &newNonce, OldValue: &oldNonce, } diff --git a/statediff/helpers.go b/statediff/builder/helpers.go similarity index 99% rename from statediff/helpers.go rename to statediff/builder/helpers.go index 8a0a6466e172..31aae926bfaf 100644 --- a/statediff/helpers.go +++ b/statediff/builder/helpers.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package builder import ( "sort" diff --git a/statediff/struct.go b/statediff/builder/struct.go similarity index 99% rename from statediff/struct.go rename to statediff/builder/struct.go index 009af4c2ea6c..7b48bba72ccf 100644 --- a/statediff/struct.go +++ b/statediff/builder/struct.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package builder import ( "encoding/json" diff --git a/statediff/extractor.go b/statediff/extractor/extractor.go similarity index 78% rename from statediff/extractor.go rename to statediff/extractor/extractor.go index 9d9b1f080c37..8910e47a1172 100644 --- a/statediff/extractor.go +++ b/statediff/extractor/extractor.go @@ -17,10 +17,12 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package extractor import ( "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/publisher" ) type Extractor interface { @@ -28,11 +30,11 @@ type Extractor interface { } type extractor struct { - Builder Builder // Interface for building state diff objects from two blocks - Publisher Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) + Builder builder.Builder // Interface for building state diff objects from two blocks + Publisher publisher.Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) } -func NewExtractor(builder Builder, publisher Publisher) (*extractor, error) { +func NewExtractor(builder builder.Builder, publisher publisher.Publisher) (*extractor, error) { return &extractor{ Builder: builder, Publisher: publisher, diff --git a/statediff/extractor/extractor_suite_test.go b/statediff/extractor/extractor_suite_test.go new file mode 100644 index 000000000000..2a3efb312807 --- /dev/null +++ b/statediff/extractor/extractor_suite_test.go @@ -0,0 +1,13 @@ +package extractor_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestExtractor(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Extractor Suite") +} diff --git a/statediff/extractor_test.go b/statediff/extractor/extractor_test.go similarity index 92% rename from statediff/extractor_test.go rename to statediff/extractor/extractor_test.go index 9ccd250f6c95..3cd8b3230cb3 100644 --- a/statediff/extractor_test.go +++ b/statediff/extractor/extractor_test.go @@ -17,30 +17,31 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff_test +package extractor_test import ( "github.com/onsi/ginkgo" - "github.com/ethereum/go-ethereum/statediff" "github.com/onsi/gomega" "github.com/ethereum/go-ethereum/core/types" "math/rand" "github.com/ethereum/go-ethereum/statediff/testhelpers" "math/big" + e "github.com/ethereum/go-ethereum/statediff/extractor" + b "github.com/ethereum/go-ethereum/statediff/builder" ) var _ = ginkgo.Describe("Extractor", func() { var publisher testhelpers.MockPublisher var builder testhelpers.MockBuilder var currentBlockNumber *big.Int var parentBlock, currentBlock *types.Block - var expectedStateDiff statediff.StateDiff - var extractor statediff.Extractor + var expectedStateDiff b.StateDiff + var extractor e.Extractor var err error ginkgo.BeforeEach(func() { publisher = testhelpers.MockPublisher{} builder = testhelpers.MockBuilder{} - extractor, err = statediff.NewExtractor(&builder, &publisher) + extractor, err = e.NewExtractor(&builder, &publisher) gomega.Expect(err).NotTo(gomega.HaveOccurred()) blockNumber := rand.Int63() @@ -49,7 +50,7 @@ var _ = ginkgo.Describe("Extractor", func() { parentBlock = types.NewBlock(&types.Header{Number: parentBlockNumber}, nil, nil, nil) currentBlock = types.NewBlock(&types.Header{Number: currentBlockNumber}, nil, nil, nil) - expectedStateDiff = statediff.StateDiff{ + expectedStateDiff = b.StateDiff{ BlockNumber: blockNumber, BlockHash: currentBlock.Hash(), CreatedAccounts: nil, diff --git a/statediff/ipfs/adder.go b/statediff/ipfs/adder.go deleted file mode 100644 index 23eea1c1ff54..000000000000 --- a/statediff/ipfs/adder.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package ipfs - -import ( - "context" - - "github.com/ipfs/go-ipfs/core" - "github.com/ipfs/go-ipfs/repo/fsrepo" - ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" -) - -type Adder interface { - Add(node ipld.Node) error -} - -type adder struct { - n *core.IpfsNode - ctx context.Context -} - -func (a adder) Add(node ipld.Node) error { - return a.n.DAG.Add(a.n.Context(), node) // For some reason DAG.Add method is not being exposed by the ipld.DAGService -} - -func NewAdder(repoPath string) (*adder, error) { - r, err := fsrepo.Open(repoPath) - if err != nil { - return nil, err - } - ctx := context.Background() - cfg := &core.BuildCfg{ - Online: false, - Repo: r, - } - ipfsNode, err := core.NewNode(ctx, cfg) - if err != nil { - return nil, err - } - return &adder{n: ipfsNode, ctx: ctx}, nil -} \ No newline at end of file diff --git a/statediff/ipfs/dag_putter.go b/statediff/ipfs/dag_putter.go deleted file mode 100644 index 4f99bdae37e2..000000000000 --- a/statediff/ipfs/dag_putter.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package ipfs - -import ( - "bytes" - "encoding/gob" - - ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/common" -) - -const ( - EthStateDiffCode = 0x99 // Register custom codec for state diff? -) - -type DagPutter interface { - DagPut(sd *statediff.StateDiff) (string, error) -} - -type dagPutter struct { - Adder -} - -func NewDagPutter(adder Adder) *dagPutter { - return &dagPutter{Adder: adder} -} - -func (bhdp *dagPutter) DagPut(sd *statediff.StateDiff) (string, error) { - nd, err := bhdp.getNode(sd) - if err != nil { - return "", err - } - err = bhdp.Add(nd) - if err != nil { - return "", err - } - return nd.Cid().String(), nil -} - -func (bhdp *dagPutter) getNode(sd *statediff.StateDiff) (ipld.Node, error) { - - var buff bytes.Buffer - enc := gob.NewEncoder(&buff) - - err := enc.Encode(sd) - if err != nil { - return nil, err - } - - raw := buff.Bytes() - cid, err := RawToCid(EthStateDiffCode, raw) - if err != nil { - return nil, err - } - - return &StateDiffNode{ - StateDiff: sd, - cid: cid, - rawdata: raw, - }, nil -} \ No newline at end of file diff --git a/statediff/ipfs/helpers.go b/statediff/ipfs/helpers.go deleted file mode 100644 index a9904afa507d..000000000000 --- a/statediff/ipfs/helpers.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package ipfs - -import ( - mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" - "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" -) - -func RawToCid(codec uint64, raw []byte) (*cid.Cid, error) { - c, err := cid.Prefix{ - Codec: codec, - Version: 1, - MhType: mh.KECCAK_256, - MhLength: -1, - }.Sum(raw) - if err != nil { - return nil, err - } - return c, nil -} \ No newline at end of file diff --git a/statediff/ipfs/node.go b/statediff/ipfs/node.go deleted file mode 100644 index dd7447a80ccf..000000000000 --- a/statediff/ipfs/node.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package ipfs - -import ( - ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" - "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" - - "github.com/i-norden/go-ethereum/statediff" -) - -type StateDiffNode struct { - *statediff.StateDiff - - cid *cid.Cid - rawdata []byte -} - -func (sdn *StateDiffNode) RawData() []byte { - return sdn.rawdata -} - -func (sdn *StateDiffNode) Cid() *cid.Cid { - return sdn.cid -} - -func (sdn StateDiffNode) String() string { - return sdn.cid.String() -} - -func (sdn StateDiffNode) Loggable() map[string]interface{} { - return sdn.cid.Loggable() -} - -func (sdn StateDiffNode) Resolve(path []string) (interface{}, []string, error) { - panic("implement me") -} - -func (sdn StateDiffNode) Tree(path string, depth int) []string { - panic("implement me") -} - -func (sdn StateDiffNode) ResolveLink(path []string) (*ipld.Link, []string, error) { - panic("implement me") -} - -func (sdn StateDiffNode) Copy() ipld.Node { - panic("implement me") -} - -func (sdn StateDiffNode) Links() []*ipld.Link { - panic("implement me") -} - -func (sdn StateDiffNode) Stat() (*ipld.NodeStat, error) { - panic("implement me") -} - -func (sdn StateDiffNode) Size() (uint64, error) { - panic("implement me") -} \ No newline at end of file diff --git a/statediff/publisher.go b/statediff/publisher/publisher.go similarity index 83% rename from statediff/publisher.go rename to statediff/publisher/publisher.go index e6526cee9d7f..a642987501bb 100644 --- a/statediff/publisher.go +++ b/statediff/publisher/publisher.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package statediff +package publisher import ( "os" @@ -25,14 +25,16 @@ import ( "time" "strconv" "strings" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff" ) type Publisher interface { - PublishStateDiff(sd *StateDiff) (string, error) + PublishStateDiff(sd *builder.StateDiff) (string, error) } type publisher struct { - Config Config + Config statediff.Config } var ( @@ -51,22 +53,22 @@ var ( updatedAccountAction = "updated" ) -func NewPublisher(config Config) (*publisher, error) { +func NewPublisher(config statediff.Config) (*publisher, error) { return &publisher{ Config: config, }, nil } -func (p *publisher) PublishStateDiff(sd *StateDiff) (string, error) { +func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { switch p.Config.Mode { - case CSV: + case statediff.CSV: return "", p.publishStateDiffToCSV(*sd) default: return "", p.publishStateDiffToCSV(*sd) } } -func (p *publisher) publishStateDiffToCSV(sd StateDiff) error { +func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) error { now := time.Now() timeStamp := now.Format(timeStampFormat) filePath := p.Config.Path + timeStamp + ".csv" @@ -102,7 +104,7 @@ func (p *publisher) publishStateDiffToCSV(sd StateDiff) error { return nil } -func accumulateUpdatedAccountRows(sd StateDiff) [][]string { +func accumulateUpdatedAccountRows(sd builder.StateDiff) [][]string { var updatedAccountRows [][]string for _, accountDiff := range sd.UpdatedAccounts { formattedAccountData := formatAccountDiffIncremental(accountDiff, sd, updatedAccountAction) @@ -113,7 +115,7 @@ func accumulateUpdatedAccountRows(sd StateDiff) [][]string { return updatedAccountRows } -func accumulateDeletedAccountRows(sd StateDiff) [][]string { +func accumulateDeletedAccountRows(sd builder.StateDiff) [][]string { var deletedAccountRows [][]string for _, accountDiff := range sd.DeletedAccounts { formattedAccountData := formatAccountDiffEventual(accountDiff, sd, deletedAccountAction) @@ -124,7 +126,7 @@ func accumulateDeletedAccountRows(sd StateDiff) [][]string { return deletedAccountRows } -func accumulateCreatedAccountRows(sd StateDiff) [][]string { +func accumulateCreatedAccountRows(sd builder.StateDiff) [][]string { var createdAccountRows [][]string for _, accountDiff := range sd.CreatedAccounts { formattedAccountData := formatAccountDiffEventual(accountDiff, sd, createdAccountAction) @@ -135,7 +137,7 @@ func accumulateCreatedAccountRows(sd StateDiff) [][]string { return createdAccountRows } -func formatAccountDiffEventual(accountDiff AccountDiffEventual, sd StateDiff, accountAction string) []string { +func formatAccountDiffEventual(accountDiff builder.AccountDiffEventual, sd builder.StateDiff, accountAction string) []string { oldContractRoot := accountDiff.ContractRoot.OldValue newContractRoot := accountDiff.ContractRoot.NewValue var storageDiffPaths []string @@ -159,7 +161,7 @@ func formatAccountDiffEventual(accountDiff AccountDiffEventual, sd StateDiff, ac return formattedAccountData } -func formatAccountDiffIncremental(accountDiff AccountDiffIncremental, sd StateDiff, accountAction string) []string { +func formatAccountDiffIncremental(accountDiff builder.AccountDiffIncremental, sd builder.StateDiff, accountAction string) []string { oldContractRoot := accountDiff.ContractRoot.OldValue newContractRoot := accountDiff.ContractRoot.NewValue var storageDiffPaths []string diff --git a/statediff/publisher/publisher_suite_test.go b/statediff/publisher/publisher_suite_test.go new file mode 100644 index 000000000000..481194b8e4ba --- /dev/null +++ b/statediff/publisher/publisher_suite_test.go @@ -0,0 +1,13 @@ +package publisher_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestPublisher(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Publisher Suite") +} diff --git a/statediff/publisher_test.go b/statediff/publisher/publisher_test.go similarity index 84% rename from statediff/publisher_test.go rename to statediff/publisher/publisher_test.go index 9d34c210749e..7becfed714f1 100644 --- a/statediff/publisher_test.go +++ b/statediff/publisher/publisher_test.go @@ -1,4 +1,4 @@ -package statediff_test +package publisher_test import ( "github.com/onsi/ginkgo" @@ -12,12 +12,14 @@ import ( "path/filepath" "strings" "strconv" + p "github.com/ethereum/go-ethereum/statediff/publisher" + "github.com/ethereum/go-ethereum/statediff/builder" ) var _ = ginkgo.Describe("Publisher", func() { ginkgo.Context("default CSV publisher", func() { var ( - publisher statediff.Publisher + publisher p.Publisher err error config = statediff.Config{ Path: "./test-", @@ -36,21 +38,21 @@ var _ = ginkgo.Describe("Publisher", func() { storagePath = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" oldStorage = "0x0" newStorage = "0x03" - storage = map[string]statediff.DiffString{storagePath: { + storage = map[string]builder.DiffString{storagePath: { NewValue: &newStorage, OldValue: &oldStorage, }} address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") - createdAccounts = map[common.Address]statediff.AccountDiffEventual{address: { - Nonce: statediff.DiffUint64{ + createdAccounts = map[common.Address]builder.AccountDiffEventual{address: { + Nonce: builder.DiffUint64{ NewValue: &newNonceValue, OldValue: &oldNonceValue, }, - Balance: statediff.DiffBigInt{ + Balance: builder.DiffBigInt{ NewValue: big.NewInt(newBalanceValue), OldValue: big.NewInt(oldBalanceValue), }, - ContractRoot: statediff.DiffString{ + ContractRoot: builder.DiffString{ NewValue: &contractRoot, OldValue: &contractRoot, }, @@ -59,33 +61,33 @@ var _ = ginkgo.Describe("Publisher", func() { Storage: storage, }} - updatedAccounts = map[common.Address]statediff.AccountDiffIncremental{address: { - Nonce: statediff.DiffUint64{ + updatedAccounts = map[common.Address]builder.AccountDiffIncremental{address: { + Nonce: builder.DiffUint64{ NewValue: &newNonceValue, OldValue: &oldNonceValue, }, - Balance: statediff.DiffBigInt{ + Balance: builder.DiffBigInt{ NewValue: big.NewInt(newBalanceValue), OldValue: big.NewInt(oldBalanceValue), }, CodeHash: codeHash, - ContractRoot: statediff.DiffString{ + ContractRoot: builder.DiffString{ NewValue: &contractRoot, OldValue: &contractRoot, }, Storage: storage, }} - deletedAccounts = map[common.Address]statediff.AccountDiffEventual{address: { - Nonce: statediff.DiffUint64{ + deletedAccounts = map[common.Address]builder.AccountDiffEventual{address: { + Nonce: builder.DiffUint64{ NewValue: &newNonceValue, OldValue: &oldNonceValue, }, - Balance: statediff.DiffBigInt{ + Balance: builder.DiffBigInt{ NewValue: big.NewInt(newBalanceValue), OldValue: big.NewInt(oldBalanceValue), }, - ContractRoot: statediff.DiffString{ + ContractRoot: builder.DiffString{ NewValue: &contractRoot, OldValue: &contractRoot, }, @@ -94,7 +96,7 @@ var _ = ginkgo.Describe("Publisher", func() { Storage: storage, }} - testStateDiff = statediff.StateDiff{ + testStateDiff = builder.StateDiff{ BlockNumber: blockNumber, BlockHash: common.HexToHash(blockHash), CreatedAccounts: createdAccounts, @@ -106,7 +108,7 @@ var _ = ginkgo.Describe("Publisher", func() { var lines [][]string var file *os.File ginkgo.BeforeEach(func() { - publisher, err = statediff.NewPublisher(config) + publisher, err = p.NewPublisher(config) gomega.Expect(err).NotTo(gomega.HaveOccurred()) _, err := publisher.PublishStateDiff(&testStateDiff) @@ -127,7 +129,7 @@ var _ = ginkgo.Describe("Publisher", func() { ginkgo.It("persists the column headers to a CSV file", func() { gomega.Expect(len(lines) > 1).To(gomega.BeTrue()) - gomega.Expect(lines[0]).To(gomega.Equal(statediff.Headers)) + gomega.Expect(lines[0]).To(gomega.Equal(p.Headers)) }) ginkgo.It("persists the created account diffs to a CSV file", func() { @@ -203,4 +205,3 @@ func getTestCSVFiles(rootPath string) []string{ gomega.Expect(err).NotTo(gomega.HaveOccurred()) return files } - diff --git a/statediff/service.go b/statediff/service/service.go similarity index 76% rename from statediff/service.go rename to statediff/service/service.go index 86839cf3b83b..d2a46c12156a 100644 --- a/statediff/service.go +++ b/statediff/service/service.go @@ -1,4 +1,4 @@ -package statediff +package service import ( "github.com/ethereum/go-ethereum/core" @@ -7,23 +7,27 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/event" "log" + e "github.com/ethereum/go-ethereum/statediff/extractor" + b "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff" + p "github.com/ethereum/go-ethereum/statediff/publisher" ) type StateDiffService struct { - builder *builder - extractor *extractor + builder *b.Builder + extractor e.Extractor blockchain *core.BlockChain } func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { - config := Config{} - builder := NewBuilder(db) - publisher, err := NewPublisher(config) + config := statediff.Config{} + builder := b.NewBuilder(db) + publisher, err := p.NewPublisher(config) if err != nil { return nil, nil } - extractor, _ := NewExtractor(builder, publisher) + extractor, _ := e.NewExtractor(builder, publisher) return &StateDiffService{ blockchain: blockChain, extractor: extractor, diff --git a/statediff/testhelpers/mocks.go b/statediff/testhelpers/mocks.go index 4bff3c02862d..8dec1d949ccc 100644 --- a/statediff/testhelpers/mocks.go +++ b/statediff/testhelpers/mocks.go @@ -2,8 +2,8 @@ package testhelpers import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff" "errors" + "github.com/ethereum/go-ethereum/statediff/builder" ) var MockError = errors.New("mock error") @@ -13,11 +13,11 @@ type MockBuilder struct { NewStateRoot common.Hash BlockNumber int64 BlockHash common.Hash - stateDiff *statediff.StateDiff + stateDiff *builder.StateDiff builderError error } -func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*statediff.StateDiff, error) { +func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*builder.StateDiff, error) { builder.OldStateRoot = oldStateRoot builder.NewStateRoot = newStateRoot builder.BlockNumber = blockNumber @@ -26,7 +26,7 @@ func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Has return builder.stateDiff, builder.builderError } -func (builder *MockBuilder) SetStateDiffToBuild(stateDiff *statediff.StateDiff) { +func (builder *MockBuilder) SetStateDiffToBuild(stateDiff *builder.StateDiff) { builder.stateDiff = stateDiff } @@ -35,11 +35,11 @@ func (builder *MockBuilder) SetBuilderError(err error) { } type MockPublisher struct{ - StateDiff *statediff.StateDiff + StateDiff *builder.StateDiff publisherError error } -func (publisher *MockPublisher) PublishStateDiff(sd *statediff.StateDiff) (string, error) { +func (publisher *MockPublisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { publisher.StateDiff = sd return "", publisher.publisherError } From ae4c889dc7bcd9470c44e0c39ca5a6c6a4fdb73a Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 13 Dec 2018 11:33:38 -0600 Subject: [PATCH 13/13] WIP - publish statediffs to IPFS --- statediff/publisher/ipfs/adder.go | 58 +++++++++++++++++++ statediff/publisher/ipfs/dag_putter.go | 80 ++++++++++++++++++++++++++ statediff/publisher/ipfs/helpers.go | 38 ++++++++++++ statediff/publisher/ipfs/node.go | 78 +++++++++++++++++++++++++ statediff/publisher/publisher.go | 28 ++++++--- 5 files changed, 274 insertions(+), 8 deletions(-) create mode 100644 statediff/publisher/ipfs/adder.go create mode 100644 statediff/publisher/ipfs/dag_putter.go create mode 100644 statediff/publisher/ipfs/helpers.go create mode 100644 statediff/publisher/ipfs/node.go diff --git a/statediff/publisher/ipfs/adder.go b/statediff/publisher/ipfs/adder.go new file mode 100644 index 000000000000..3962ddc3a852 --- /dev/null +++ b/statediff/publisher/ipfs/adder.go @@ -0,0 +1,58 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "context" + + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/repo/fsrepo" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +type Adder interface { + Add(node ipld.Node) error +} + +type adder struct { + n *core.IpfsNode + ctx context.Context +} + +func (a adder) Add(node ipld.Node) error { + return a.n.DAG.Add(a.n.Context(), node) // For some reason DAG.Add method is not being exposed by the ipld.DAGService +} + +func NewAdder(repoPath string) (*adder, error) { + r, err := fsrepo.Open(repoPath) + if err != nil { + return nil, err + } + ctx := context.Background() + cfg := &core.BuildCfg{ + Online: false, + Repo: r, + } + ipfsNode, err := core.NewNode(ctx, cfg) + if err != nil { + return nil, err + } + return &adder{n: ipfsNode, ctx: ctx}, nil +} diff --git a/statediff/publisher/ipfs/dag_putter.go b/statediff/publisher/ipfs/dag_putter.go new file mode 100644 index 000000000000..e44d8b9e0f20 --- /dev/null +++ b/statediff/publisher/ipfs/dag_putter.go @@ -0,0 +1,80 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "bytes" + "encoding/gob" + + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + + "github.com/ethereum/go-ethereum/statediff/builder" +) + +const ( + EthStateDiffCode = 0x99 // Register custom codec for state diff? +) + +type DagPutter interface { + DagPut(sd *builder.StateDiff) (string, error) +} + +type dagPutter struct { + Adder +} + +func NewDagPutter(adder Adder) *dagPutter { + return &dagPutter{Adder: adder} +} + +func (bhdp *dagPutter) DagPut(sd *builder.StateDiff) (string, error) { + nd, err := bhdp.getNode(sd) + if err != nil { + return "", err + } + err = bhdp.Add(nd) + if err != nil { + return "", err + } + return nd.Cid().String(), nil +} + +func (bhdp *dagPutter) getNode(sd *builder.StateDiff) (ipld.Node, error) { + + var buff bytes.Buffer + enc := gob.NewEncoder(&buff) + + err := enc.Encode(sd) + if err != nil { + return nil, err + } + + raw := buff.Bytes() + cid, err := RawToCid(EthStateDiffCode, raw) + if err != nil { + return nil, err + } + + return &StateDiffNode{ + StateDiff: sd, + cid: cid, + rawdata: raw, + }, nil +} diff --git a/statediff/publisher/ipfs/helpers.go b/statediff/publisher/ipfs/helpers.go new file mode 100644 index 000000000000..36c9fd7b8317 --- /dev/null +++ b/statediff/publisher/ipfs/helpers.go @@ -0,0 +1,38 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" +) + +func RawToCid(codec uint64, raw []byte) (*cid.Cid, error) { + c, err := cid.Prefix{ + Codec: codec, + Version: 1, + MhType: mh.KECCAK_256, + MhLength: -1, + }.Sum(raw) + if err != nil { + return nil, err + } + return &c, nil +} diff --git a/statediff/publisher/ipfs/node.go b/statediff/publisher/ipfs/node.go new file mode 100644 index 000000000000..0785d91331c1 --- /dev/null +++ b/statediff/publisher/ipfs/node.go @@ -0,0 +1,78 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + + "github.com/ethereum/go-ethereum/statediff/builder" +) + +type StateDiffNode struct { + *builder.StateDiff + + cid *cid.Cid + rawdata []byte +} + +func (sdn *StateDiffNode) RawData() []byte { + return sdn.rawdata +} + +func (sdn *StateDiffNode) Cid() cid.Cid { + return *sdn.cid +} + +func (sdn StateDiffNode) String() string { + return sdn.cid.String() +} + +func (sdn StateDiffNode) Loggable() map[string]interface{} { + return sdn.cid.Loggable() +} + +func (sdn StateDiffNode) Resolve(path []string) (interface{}, []string, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Tree(path string, depth int) []string { + panic("implement me") +} + +func (sdn StateDiffNode) ResolveLink(path []string) (*ipld.Link, []string, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Copy() ipld.Node { + panic("implement me") +} + +func (sdn StateDiffNode) Links() []*ipld.Link { + panic("implement me") +} + +func (sdn StateDiffNode) Stat() (*ipld.NodeStat, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Size() (uint64, error) { + panic("implement me") +} diff --git a/statediff/publisher/publisher.go b/statediff/publisher/publisher.go index a642987501bb..dc62e8a61ecb 100644 --- a/statediff/publisher/publisher.go +++ b/statediff/publisher/publisher.go @@ -20,13 +20,14 @@ package publisher import ( - "os" "encoding/csv" - "time" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/publisher/ipfs" + "os" "strconv" "strings" - "github.com/ethereum/go-ethereum/statediff/builder" - "github.com/ethereum/go-ethereum/statediff" + "time" ) type Publisher interface { @@ -34,6 +35,7 @@ type Publisher interface { } type publisher struct { + ipfs.DagPutter Config statediff.Config } @@ -47,15 +49,20 @@ var ( "storageDiffPaths", } - timeStampFormat = "20060102150405.00000" + timeStampFormat = "20060102150405.00000" deletedAccountAction = "deleted" createdAccountAction = "created" updatedAccountAction = "updated" ) func NewPublisher(config statediff.Config) (*publisher, error) { + adder, err := ipfs.NewAdder(config.Path) + if err != nil { + return nil, err + } return &publisher{ - Config: config, + DagPutter: ipfs.NewDagPutter(adder), + Config: config, }, nil } @@ -63,6 +70,12 @@ func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { switch p.Config.Mode { case statediff.CSV: return "", p.publishStateDiffToCSV(*sd) + case statediff.IPLD: + cidStr, err := p.DagPut(sd) + if err != nil { + return "", err + } + return cidStr, err default: return "", p.publishStateDiffToCSV(*sd) } @@ -94,7 +107,7 @@ func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) error { data = append(data, row) } - for _, value := range data{ + for _, value := range data { err := writer.Write(value) if err != nil { return err @@ -184,4 +197,3 @@ func formatAccountDiffIncremental(accountDiff builder.AccountDiffIncremental, sd } return formattedAccountData } -