diff --git a/cmd/geth/config.go b/cmd/geth/config.go index b0749d23291b..a05e78b112e6 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -178,6 +178,10 @@ func makeFullNode(ctx *cli.Context) *node.Node { if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, cfg.Ethstats.URL) } + + if ctx.GlobalBool(utils.StateDiffFlag.Name) { + utils.RegisterStateDiffService(stack) + } return stack } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 69802a48a763..15f76554851c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -132,6 +132,7 @@ var ( utils.GpoPercentileFlag, utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, + utils.StateDiffFlag, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 82f17e0ee895..152ac059b0c0 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -245,6 +245,12 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerLegacyExtraDataFlag, }, }, + { + Name: "STATE DIFF", + Flags: []cli.Flag{ + utils.StateDiffFlag, + }, + }, { Name: "MISC", }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d7b698c7e2e5..54e3dd83d1ba 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -58,6 +58,7 @@ import ( "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -626,6 +627,12 @@ var ( Usage: "External EVM configuration (default = built-in interpreter)", Value: "", } + + // Statediff flags + StateDiffFlag = cli.BoolFlag{ + Name: "statediff", + Usage: "Enables the calculation of state diffs between each block, persists these state diffs in ipfs", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1321,6 +1328,18 @@ func RegisterEthStatsService(stack *node.Node, url string) { } } +func RegisterStateDiffService(stack *node.Node) { + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + var ethServ *eth.Ethereum + ctx.Service(ðServ) + chainDb := ethServ.ChainDb() + blockChain := ethServ.BlockChain() + return statediff.NewStateDiffService(chainDb, blockChain) + }); err != nil { + Fatalf("Failed to register State Diff Service", err) + } +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/core/blockchain.go b/core/blockchain.go index d173b2de294d..1b48020a6c15 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -174,6 +174,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) var err error + bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { return nil, err @@ -1187,6 +1188,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty parent = chain[i-1] } state, err := state.New(parent.Root(), bc.stateCache) + if err != nil { return i, events, coalescedLogs, err } diff --git a/eth/backend.go b/eth/backend.go index 4721408425cb..0de0a19803f2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -154,7 +154,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { EWASMInterpreter: config.EWASMInterpreter, EVMInterpreter: config.EVMInterpreter, } - cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout} + cacheConfig = &core.CacheConfig{ + Disabled: config.NoPruning, + TrieCleanLimit: config.TrieCleanCache, + TrieDirtyLimit: config.TrieDirtyCache, + TrieTimeLimit: config.TrieTimeout, + } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) if err != nil { diff --git a/statediff/builder/builder.go b/statediff/builder/builder.go new file mode 100644 index 000000000000..30479a60cfe0 --- /dev/null +++ b/statediff/builder/builder.go @@ -0,0 +1,326 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +type Builder interface { + BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) +} + +type builder struct { + chainDB ethdb.Database + trieDB *trie.Database + cachedTrie *trie.Trie +} + +func NewBuilder(db ethdb.Database) *builder { + return &builder{ + chainDB: db, + trieDB: trie.NewDatabase(db), + } +} + +func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { + // Generate tries for old and new states + oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) + if err != nil { + log.Debug("error creating oldTrie", err) + //getting this error: error creating oldTrie missing trie node ddfbb83966d870891aa47147269447a83564d1defaefad5f9844a3a3a2a08433 (path ) + return nil, err + } + newTrie, err := trie.New(newStateRoot, sdb.trieDB) + if err != nil { + log.Debug("error creating newTrie", err) + return nil, err + } + + // Find created accounts + oldIt := oldTrie.NodeIterator([]byte{}) + newIt := newTrie.NodeIterator([]byte{}) + creations, err := sdb.collectDiffNodes(oldIt, newIt) + if err != nil { + log.Debug("error collecting creation diff nodes", err) + return nil, err + } + + // Find deleted accounts + oldIt = oldTrie.NodeIterator(make([]byte, 0)) + newIt = newTrie.NodeIterator(make([]byte, 0)) + deletions, err := sdb.collectDiffNodes(newIt, oldIt) + if err != nil { + log.Debug("error collecting deletion diff nodes", err) + return nil, err + } + + // Find all the diffed keys + createKeys := sortKeys(creations) + deleteKeys := sortKeys(deletions) + updatedKeys := findIntersection(createKeys, deleteKeys) + + // Build and return the statediff + updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) + if err != nil { + log.Debug("error building diff incremental for updated", err) + return nil, err + } + createdAccounts, err := sdb.buildDiffEventual(creations, true) + if err != nil { + log.Debug("error building diff incremental for created", err) + return nil, err + } + deletedAccounts, err := sdb.buildDiffEventual(deletions, false) + if err != nil { + log.Debug("error building diff incremental for deleted", err) + return nil, err + } + + return &StateDiff{ + BlockNumber: blockNumber, + BlockHash: blockHash, + CreatedAccounts: createdAccounts, + DeletedAccounts: deletedAccounts, + UpdatedAccounts: updatedAccounts, + }, nil +} + +func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { + var diffAccounts = make(map[common.Address]*state.Account) + it, _ := trie.NewDifferenceIterator(a, b) + + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", common.Hash(it.Hash())) + if it.Leaf() { + + // lookup address + path := make([]byte, len(it.Path())-1) + copy(path, it.Path()) + addr, err := sdb.addressByPath(path) + if err != nil { + log.Error("Error looking up address via path", "path", path, "error", err) + return nil, err + } + + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil { + log.Error("Error looking up account via address", "address", addr, "error", err) + return nil, err + } + + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", addr, "account", account) + diffAccounts[*addr] = &account + } + cont := it.Next(true) + if !cont { + break + } + } + + return diffAccounts, nil +} + +func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account, created bool) (map[common.Address]AccountDiffEventual, error) { + accountDiffs := make(map[common.Address]AccountDiffEventual) + for addr, val := range accounts { + sr := val.Root + storageDiffs, err := sdb.buildStorageDiffsEventual(sr, created) + if err != nil { + log.Error("Failed building eventual storage diffs", "Address", val, "error", err) + return nil, err + } + + codeBytes, err := sdb.chainDB.Get(val.CodeHash) + + codeHash := common.ToHex(val.CodeHash) + hexRoot := val.Root.Hex() + + if created { + nonce := DiffUint64{ + NewValue: &val.Nonce, + } + + balance := DiffBigInt{ + NewValue: val.Balance, + } + + contractRoot := DiffString{ + NewValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + Code: codeBytes, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + } else { + nonce := DiffUint64{ + OldValue: &val.Nonce, + } + balance := DiffBigInt{ + OldValue: val.Balance, + } + contractRoot := DiffString{ + OldValue: &hexRoot, + } + accountDiffs[addr] = AccountDiffEventual{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + } + } + + return accountDiffs, nil +} + +func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys []string) (map[common.Address]AccountDiffIncremental, error) { + updatedAccounts := make(map[common.Address]AccountDiffIncremental) + for _, val := range updatedKeys { + createdAcc := creations[common.HexToAddress(val)] + deletedAcc := deletions[common.HexToAddress(val)] + oldSR := deletedAcc.Root + newSR := createdAcc.Root + if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil { + log.Error("Failed building storage diffs", "Address", val, "error", err) + return nil, err + } else { + nonce := DiffUint64{ + NewValue: &createdAcc.Nonce, + OldValue: &deletedAcc.Nonce, + } + + balance := DiffBigInt{ + NewValue: createdAcc.Balance, + OldValue: deletedAcc.Balance, + } + codeHash := common.ToHex(createdAcc.CodeHash) + + nHexRoot := createdAcc.Root.Hex() + oHexRoot := deletedAcc.Root.Hex() + contractRoot := DiffString{ + NewValue: &nHexRoot, + OldValue: &oHexRoot, + } + + updatedAccounts[common.HexToAddress(val)] = AccountDiffIncremental{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + delete(creations, common.HexToAddress(val)) + delete(deletions, common.HexToAddress(val)) + } + } + return updatedAccounts, nil +} + +func (sdb *builder) buildStorageDiffsEventual(sr common.Hash, creation bool) (map[string]DiffString, error) { + log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) + sTrie, err := trie.New(sr, sdb.trieDB) + if err != nil { + return nil, err + } + it := sTrie.NodeIterator(make([]byte, 0)) + storageDiffs := make(map[string]DiffString) + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + path := pathToStr(it) + value := common.ToHex(it.LeafBlob()) + if creation { + storageDiffs[path] = DiffString{NewValue: &value} + } else { + storageDiffs[path] = DiffString{OldValue: &value} + } + } + cont := it.Next(true) + if !cont { + break + } + } + return storageDiffs, nil +} + +func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]DiffString, error) { + log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) + oldTrie, err := trie.New(oldSR, sdb.trieDB) + if err != nil { + return nil, err + } + newTrie, err := trie.New(newSR, sdb.trieDB) + if err != nil { + return nil, err + } + + oldIt := oldTrie.NodeIterator(make([]byte, 0)) + newIt := newTrie.NodeIterator(make([]byte, 0)) + it, _ := trie.NewDifferenceIterator(oldIt, newIt) + storageDiffs := make(map[string]DiffString) + for { + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + path := pathToStr(it) + value := common.ToHex(it.LeafBlob()) + if oldVal, err := oldTrie.TryGet(it.LeafKey()); err != nil { + log.Error("Failed to look up value in oldTrie", "path", path, "error", err) + } else { + hexOldVal := common.ToHex(oldVal) + storageDiffs[path] = DiffString{OldValue: &hexOldVal, NewValue: &value} + } + } + + cont := it.Next(true) + if !cont { + break + } + } + return storageDiffs, nil +} + +func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { + // db := core.PreimageTable(sdb.chainDb) + log.Debug("Looking up address from path", "path", common.ToHex(append([]byte("secure-key-"), path...))) + // if addrBytes,err := db.Get(path); err != nil { + if addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeybytes(path)...)); err != nil { + log.Error("Error looking up address via path", "path", common.ToHex(append([]byte("secure-key-"), path...)), "error", err) + return nil, err + } else { + addr := common.BytesToAddress(addrBytes) + log.Debug("Address found", "Address", addr) + return &addr, nil + } + +} diff --git a/statediff/builder/builder_suite_test.go b/statediff/builder/builder_suite_test.go new file mode 100644 index 000000000000..bbbf63078e7a --- /dev/null +++ b/statediff/builder/builder_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiffBuilder(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Builder Suite") +} \ No newline at end of file diff --git a/statediff/builder/builder_test.go b/statediff/builder/builder_test.go new file mode 100644 index 000000000000..793a6347d419 --- /dev/null +++ b/statediff/builder/builder_test.go @@ -0,0 +1,322 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder_test + +import ( + "math/big" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/crypto" + statediff "github.com/ethereum/go-ethereum/statediff/builder" +) + + +var ( + testdb = ethdb.NewMemDatabase() + + testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + testBankFunds = big.NewInt(100000000) + genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) + + account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + + contractCode = common.Hex2Bytes("606060405260cc8060106000396000f360606040526000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146041578063c16431b914606b57603f565b005b6055600480803590602001909190505060a9565b6040518082815260200191505060405180910390f35b60886004808035906020019091908035906020019091905050608a565b005b80600060005083606481101560025790900160005b50819055505b5050565b6000600060005082606481101560025790900160005b5054905060c7565b91905056") + contractAddr common.Address + + emptyAccountDiffEventualMap = make(map[common.Address]statediff.AccountDiffEventual) + emptyAccountDiffIncrementalMap = make(map[common.Address]statediff.AccountDiffIncremental) +) +/* +contract test { + + uint256[100] data; + + function Put(uint256 addr, uint256 value) { + data[addr] = value; + } + + function Get(uint256 addr) constant returns (uint256 value) { + return data[addr]; + } +} +*/ + + +// makeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 3rd block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { + blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, testChainGen) + hashes := make([]common.Hash, n+1) + hashes[len(hashes)-1] = parent.Hash() + blockm := make(map[common.Hash]*types.Block, n+1) + blockm[parent.Hash()] = parent + for i, b := range blocks { + hashes[len(hashes)-i-2] = b.Hash() + blockm[b.Hash()] = b + } + return hashes, blockm +} + + +func testChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // account1Addr passes it on to account #2. + // account1Addr creates a test contract. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + nonce := block.TxNonce(account1Addr) + tx2, _ := types.SignTx(types.NewTransaction(nonce, account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, account1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), contractCode), signer, account1Key) + contractAddr = crypto.CreateAddress(account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592 + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(account2Addr) + //get function: 60cd2685 + //put function: c16431b9 + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), contractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) + block.AddTx(tx) + } +} +var _ = ginkgo.FDescribe("", func() { + var ( + block0Hash, block1Hash, block2Hash, block3Hash common.Hash + block0, block1, block2, block3 *types.Block + builder statediff.Builder + miningReward = int64(3000000000000000000) + burnAddress = common.HexToAddress("0x0") + diff *statediff.StateDiff + err error + ) + + ginkgo.BeforeEach(func() { + _, blocks := makeChain(3, 0, genesis) + block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash = common.HexToHash("0x47c398dd688eaa4dd11b006888156783fe32df83d59b197c0fcd303408103d39") + block2Hash = common.HexToHash("0x351b2f531838683ba457e8ca4d3a844cc48147dceafbcb589dc6e3227856ee75") + block3Hash = common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73") + + block0 = blocks[block0Hash] + block1 = blocks[block1Hash] + block2 = blocks[block2Hash] + block3 = blocks[block3Hash] + builder = statediff.NewBuilder(testdb) + }) + + ginkgo.It("returns empty account diff collections when the state root hasn't changed", func() { + expectedDiff := statediff.StateDiff{ + BlockNumber: block0.Number().Int64(), + BlockHash: block0Hash, + CreatedAccounts: emptyAccountDiffEventualMap, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: emptyAccountDiffIncrementalMap, + } + + diff, err := builder.BuildStateDiff(block0.Root(), block0.Root(), block0.Number().Int64(), block0Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(diff).To(gomega.Equal(&expectedDiff)) + }) + + ginkgo.Context("Block 1", func() { + //10000 transferred from testBankAddress to account1Addr + var balanceChange = int64(10000) + + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block0.Root(), block1.Root(), block1.Number().Int64(), block1Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block1.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block1Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance diffs for updated accounts", func() { + expectedBankBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(testBankFunds.Int64() - balanceChange), + OldValue: testBankFunds, + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(1)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Balance).To(gomega.Equal(expectedBankBalanceDiff)) + }) + + ginkgo.It("returns balance diffs for new accounts", func() { + expectedAccount1BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(balanceChange), + OldValue: nil, + } + + expectedBurnAddrBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(miningReward), + OldValue: nil, + } + + gomega.Expect(len(diff.CreatedAccounts)).To(gomega.Equal(2)) + gomega.Expect(diff.CreatedAccounts[account1Addr].Balance).To(gomega.Equal(expectedAccount1BalanceDiff)) + gomega.Expect(diff.CreatedAccounts[burnAddress].Balance).To(gomega.Equal(expectedBurnAddrBalanceDiff)) + }) + }) + + ginkgo.Context("Block 2", func() { + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + var ( + balanceChange = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + ) + + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block1.Root(), block2.Root(), block2.Number().Int64(), block2Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block2.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block2Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance diffs for updated accounts", func() { + expectedBankBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block1BankBalance - balanceChange), + OldValue: big.NewInt(block1BankBalance), + } + + expectedAccount1BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block1Account1Balance - balanceChange + balanceChange), + OldValue: big.NewInt(block1Account1Balance), + } + + expectedBurnBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(miningReward + miningReward), + OldValue: big.NewInt(miningReward), + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(3)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Balance).To(gomega.Equal(expectedBankBalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[account1Addr].Balance).To(gomega.Equal(expectedAccount1BalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[burnAddress].Balance).To(gomega.Equal(expectedBurnBalanceDiff)) + }) + + ginkgo.It("returns balance diffs for new accounts", func() { + expectedAccount2BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(balanceChange), + OldValue: nil, + } + + expectedContractBalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(0), + OldValue: nil, + } + + gomega.Expect(len(diff.CreatedAccounts)).To(gomega.Equal(2)) + gomega.Expect(diff.CreatedAccounts[account2Addr].Balance).To(gomega.Equal(expectedAccount2BalanceDiff)) + gomega.Expect(diff.CreatedAccounts[contractAddr].Balance).To(gomega.Equal(expectedContractBalanceDiff)) + }) + }) + + ginkgo.Context("Block 3", func() { + //the contract's storage is changed + //and the block is mined by account 2 + ginkgo.BeforeEach(func() { + diff, err = builder.BuildStateDiff(block2.Root(), block3.Root(), block3.Number().Int64(), block3Hash) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("includes the block number and hash", func() { + gomega.Expect(diff.BlockNumber).To(gomega.Equal(block3.Number().Int64())) + gomega.Expect(diff.BlockHash).To(gomega.Equal(block3Hash)) + }) + + ginkgo.It("returns an empty collection for deleted accounts", func() { + gomega.Expect(diff.DeletedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns an empty collection for created accounts", func() { + gomega.Expect(diff.CreatedAccounts).To(gomega.Equal(emptyAccountDiffEventualMap)) + }) + + ginkgo.It("returns balance, storage and nonce diffs for updated accounts", func() { + block2Account2Balance := int64(1000) + expectedAcct2BalanceDiff := statediff.DiffBigInt{ + NewValue: big.NewInt(block2Account2Balance + miningReward), + OldValue: big.NewInt(block2Account2Balance), + } + + expectedContractStorageDiff := make(map[string]statediff.DiffString) + newVal := "0x03" + oldVal := "0x0" + path := "0x405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace" + expectedContractStorageDiff[path] = statediff.DiffString{ + NewValue: &newVal, + OldValue: &oldVal, + } + + oldNonce := uint64(2) + newNonce := uint64(3) + expectedBankNonceDiff := statediff.DiffUint64{ + NewValue: &newNonce, + OldValue: &oldNonce, + } + + gomega.Expect(len(diff.UpdatedAccounts)).To(gomega.Equal(3)) + gomega.Expect(diff.UpdatedAccounts[account2Addr].Balance).To(gomega.Equal(expectedAcct2BalanceDiff)) + gomega.Expect(diff.UpdatedAccounts[contractAddr].Storage).To(gomega.Equal(expectedContractStorageDiff)) + gomega.Expect(diff.UpdatedAccounts[testBankAddress].Nonce).To(gomega.Equal(expectedBankNonceDiff)) + }) + }) +}) \ No newline at end of file diff --git a/statediff/builder/helpers.go b/statediff/builder/helpers.go new file mode 100644 index 000000000000..31aae926bfaf --- /dev/null +++ b/statediff/builder/helpers.go @@ -0,0 +1,118 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder + +import ( + "sort" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/trie" +) + +func sortKeys(data map[common.Address]*state.Account) []string { + var keys []string + for key := range data { + keys = append(keys, key.Hex()) + } + sort.Strings(keys) + + return keys +} + +func findIntersection(a, b []string) []string { + lenA := len(a) + lenB := len(b) + iOfA, iOfB := 0, 0 + updates := make([]string, 0) + if iOfA >= lenA || iOfB >= lenB { + return updates + } + for { + switch strings.Compare(a[iOfA], b[iOfB]) { + // a[iOfA] < b[iOfB] + case -1: + iOfA++ + if iOfA >= lenA { + return updates + } + break + // a[iOfA] == b[iOfB] + case 0: + updates = append(updates, a[iOfA]) + iOfA++ + iOfB++ + if iOfA >= lenA || iOfB >= lenB { + return updates + } + break + // a[iOfA] > b[iOfB] + case 1: + iOfB++ + if iOfB >= lenB { + return updates + } + break + } + } + +} + +func pathToStr(it trie.NodeIterator) string { + path := it.Path() + if hasTerm(path) { + path = path[:len(path)-1] + } + nibblePath := "" + for i, v := range common.ToHex(path) { + if i%2 == 0 && i > 1 { + continue + } + nibblePath = nibblePath + string(v) + } + + return nibblePath +} + +// Duplicated from trie/encoding.go +func hexToKeybytes(hex []byte) []byte { + if hasTerm(hex) { + hex = hex[:len(hex)-1] + } + if len(hex)&1 != 0 { + panic("can't convert hex key of odd length") + } + key := make([]byte, (len(hex)+1)/2) + decodeNibbles(hex, key) + + return key +} + +func decodeNibbles(nibbles []byte, bytes []byte) { + for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { + bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] + } +} + +// hasTerm returns whether a hex key has the terminator flag. +func hasTerm(s []byte) bool { + return len(s) > 0 && s[len(s)-1] == 16 +} \ No newline at end of file diff --git a/statediff/builder/struct.go b/statediff/builder/struct.go new file mode 100644 index 000000000000..7b48bba72ccf --- /dev/null +++ b/statediff/builder/struct.go @@ -0,0 +1,86 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +type StateDiff struct { + BlockNumber int64 `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts map[common.Address]AccountDiffEventual `json:"createdAccounts" gencodec:"required"` + DeletedAccounts map[common.Address]AccountDiffEventual `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts map[common.Address]AccountDiffIncremental `json:"updatedAccounts" gencodec:"required"` + + encoded []byte + err error +} + +func (self *StateDiff) ensureEncoded() { + if self.encoded == nil && self.err == nil { + self.encoded, self.err = json.Marshal(self) + } +} + +// Implement Encoder interface for StateDiff +func (sd *StateDiff) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Implement Encoder interface for StateDiff +func (sd *StateDiff) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err +} + +type AccountDiffEventual struct { + Nonce DiffUint64 `json:"nonce" gencodec:"required"` + Balance DiffBigInt `json:"balance" gencodec:"required"` + Code []byte `json:"code" gencodec:"required"` + CodeHash string `json:"codeHash" gencodec:"required"` + ContractRoot DiffString `json:"contractRoot" gencodec:"required"` + Storage map[string]DiffString `json:"storage" gencodec:"required"` +} + +type AccountDiffIncremental struct { + Nonce DiffUint64 `json:"nonce" gencodec:"required"` + Balance DiffBigInt `json:"balance" gencodec:"required"` + CodeHash string `json:"codeHash" gencodec:"required"` + ContractRoot DiffString `json:"contractRoot" gencodec:"required"` + Storage map[string]DiffString `json:"storage" gencodec:"required"` +} + +type DiffString struct { + NewValue *string `json:"newValue" gencodec:"optional"` + OldValue *string `json:"oldValue" gencodec:"optional"` +} +type DiffUint64 struct { + NewValue *uint64 `json:"newValue" gencodec:"optional"` + OldValue *uint64 `json:"oldValue" gencodec:"optional"` +} +type DiffBigInt struct { + NewValue *big.Int `json:"newValue" gencodec:"optional"` + OldValue *big.Int `json:"oldValue" gencodec:"optional"` +} diff --git a/statediff/config.go b/statediff/config.go new file mode 100644 index 000000000000..ac207acd1347 --- /dev/null +++ b/statediff/config.go @@ -0,0 +1,81 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import "fmt" + +type Config struct { + On bool // Whether or not to extract state diffs + Mode StateDiffMode // Mode for storing diffs + Path string // Path for storing diffs +} + +type StateDiffMode int + +const ( + IPLD StateDiffMode = iota + LDB + SQL +) + +func (mode StateDiffMode) IsValid() bool { + return mode >= IPLD && mode <= SQL +} + +// String implements the stringer interface. +func (mode StateDiffMode) String() string { + switch mode { + case IPLD: + return "ipfs" + case LDB: + return "ldb" + case SQL: + return "sql" + default: + return "unknown" + } +} + +func (mode StateDiffMode) MarshalText() ([]byte, error) { + switch mode { + case IPLD: + return []byte("ipfs"), nil + case LDB: + return []byte("ldb"), nil + case SQL: + return []byte("sql"), nil + default: + return nil, fmt.Errorf("unknown state diff storage mode %d", mode) + } +} + +func (mode *StateDiffMode) UnmarshalText(text []byte) error { + switch string(text) { + case "ipfs": + *mode = IPLD + case "ldb": + *mode = LDB + case "sql": + *mode = SQL + default: + return fmt.Errorf(`unknown state diff storage mode %q, want "ipfs", "ldb" or "sql"`, text) + } + return nil +} \ No newline at end of file diff --git a/statediff/extractor/extractor.go b/statediff/extractor/extractor.go new file mode 100644 index 000000000000..d485f5e13067 --- /dev/null +++ b/statediff/extractor/extractor.go @@ -0,0 +1,51 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package extractor + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/publisher" +) + +type Extractor interface { + ExtractStateDiff(parent, current types.Block) (string, error) +} + +type extractor struct { + Builder builder.Builder // Interface for building state diff objects from two blocks + Publisher publisher.Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) +} + +func NewExtractor(b builder.Builder, p publisher.Publisher) (*extractor, error) { + return &extractor{ + Builder: b, + Publisher: p, + }, nil +} + +func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) { + stateDiff, err := e.Builder.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) + if err != nil { + return "", err + } + + return e.Publisher.PublishStateDiff(stateDiff) +} \ No newline at end of file diff --git a/statediff/extractor/extractor_suite_test.go b/statediff/extractor/extractor_suite_test.go new file mode 100644 index 000000000000..fe7719032769 --- /dev/null +++ b/statediff/extractor/extractor_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package extractor_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiffExtractor(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Extractor Suite") +} \ No newline at end of file diff --git a/statediff/extractor/extractor_test.go b/statediff/extractor/extractor_test.go new file mode 100644 index 000000000000..d76329726452 --- /dev/null +++ b/statediff/extractor/extractor_test.go @@ -0,0 +1,102 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package extractor_test + +import ( + "math/big" + "math/rand" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + + "github.com/ethereum/go-ethereum/statediff/extractor" + statediff "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/testhelpers" +) +var _ = ginkgo.Describe("Extractor", func() { + var publisher testhelpers.MockPublisher + var builder testhelpers.MockBuilder + var currentBlockNumber *big.Int + var parentBlock, currentBlock *types.Block + var expectedStateDiff statediff.StateDiff + var ex extractor.Extractor + var err error + + ginkgo.BeforeEach(func() { + publisher = testhelpers.MockPublisher{} + builder = testhelpers.MockBuilder{} + ex, err = extractor.NewExtractor(&builder, &publisher) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + blockNumber := rand.Int63() + parentBlockNumber := big.NewInt(blockNumber - int64(1)) + currentBlockNumber = big.NewInt(blockNumber) + parentBlock = types.NewBlock(&types.Header{Number: parentBlockNumber}, nil, nil, nil) + currentBlock = types.NewBlock(&types.Header{Number: currentBlockNumber}, nil, nil, nil) + + expectedStateDiff = statediff.StateDiff{ + BlockNumber: blockNumber, + BlockHash: currentBlock.Hash(), + CreatedAccounts: nil, + DeletedAccounts: nil, + UpdatedAccounts: nil, + } + }) + + ginkgo.It("builds a state diff struct", func() { + builder.SetStateDiffToBuild(&expectedStateDiff) + + _, err = ex.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(builder.OldStateRoot).To(gomega.Equal(parentBlock.Root())) + gomega.Expect(builder.NewStateRoot).To(gomega.Equal(currentBlock.Root())) + gomega.Expect(builder.BlockNumber).To(gomega.Equal(currentBlockNumber.Int64())) + gomega.Expect(builder.BlockHash).To(gomega.Equal(currentBlock.Hash())) + }) + + ginkgo.It("returns an error if building the state diff fails", func() { + builder.SetBuilderError(testhelpers.MockError) + + _, err = ex.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err).To(gomega.MatchError(testhelpers.MockError)) + }) + + ginkgo.It("publishes the state diff struct", func() { + builder.SetStateDiffToBuild(&expectedStateDiff) + + _, err = ex.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(publisher.StateDiff).To(gomega.Equal(&expectedStateDiff)) + }) + + ginkgo.It("returns an error if publishing the diff fails", func() { + publisher.SetPublisherError(testhelpers.MockError) + + _, err = ex.ExtractStateDiff(*parentBlock, *currentBlock) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err).To(gomega.MatchError(testhelpers.MockError)) + }) +}) diff --git a/statediff/publisher/ipfs/adder.go b/statediff/publisher/ipfs/adder.go new file mode 100644 index 000000000000..eb8e012c53a1 --- /dev/null +++ b/statediff/publisher/ipfs/adder.go @@ -0,0 +1,58 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "context" + + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/repo/fsrepo" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +type Adder interface { + Add(node ipld.Node) error +} + +type adder struct { + n *core.IpfsNode + ctx context.Context +} + +func (a adder) Add(node ipld.Node) error { + return a.n.DAG.Add(a.n.Context(), node) // For some reason DAG.Add method is not being exposed by the ipld.DAGService +} + +func NewAdder(repoPath string) (*adder, error) { + r, err := fsrepo.Open(repoPath) + if err != nil { + return nil, err + } + ctx := context.Background() + cfg := &core.BuildCfg{ + Online: false, + Repo: r, + } + ipfsNode, err := core.NewNode(ctx, cfg) + if err != nil { + return nil, err + } + return &adder{n: ipfsNode, ctx: ctx}, nil +} \ No newline at end of file diff --git a/statediff/publisher/ipfs/dag_putter.go b/statediff/publisher/ipfs/dag_putter.go new file mode 100644 index 000000000000..2f2ff02e2eef --- /dev/null +++ b/statediff/publisher/ipfs/dag_putter.go @@ -0,0 +1,80 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "bytes" + "encoding/gob" + + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + + "github.com/ethereum/go-ethereum/statediff/builder" +) + +const ( + EthStateDiffCode = 0x99 // Register custom codec for state diff? +) + +type DagPutter interface { + DagPut(sd *builder.StateDiff) (string, error) +} + +type dagPutter struct { + Adder +} + +func NewDagPutter(adder Adder) *dagPutter { + return &dagPutter{Adder: adder} +} + +func (bhdp *dagPutter) DagPut(sd *builder.StateDiff) (string, error) { + nd, err := bhdp.getNode(sd) + if err != nil { + return "", err + } + err = bhdp.Add(nd) + if err != nil { + return "", err + } + return nd.Cid().String(), nil +} + +func (bhdp *dagPutter) getNode(sd *builder.StateDiff) (ipld.Node, error) { + + var buff bytes.Buffer + enc := gob.NewEncoder(&buff) + + err := enc.Encode(sd) + if err != nil { + return nil, err + } + + raw := buff.Bytes() + cid, err := RawToCid(EthStateDiffCode, raw) + if err != nil { + return nil, err + } + + return &StateDiffNode{ + StateDiff: sd, + cid: cid, + rawdata: raw, + }, nil +} \ No newline at end of file diff --git a/statediff/publisher/ipfs/helpers.go b/statediff/publisher/ipfs/helpers.go new file mode 100644 index 000000000000..ea6ca5ab5c78 --- /dev/null +++ b/statediff/publisher/ipfs/helpers.go @@ -0,0 +1,38 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" +) + +func RawToCid(codec uint64, raw []byte) (*cid.Cid, error) { + c, err := cid.Prefix{ + Codec: codec, + Version: 1, + MhType: mh.KECCAK_256, + MhLength: -1, + }.Sum(raw) + if err != nil { + return nil, err + } + return &c, nil +} \ No newline at end of file diff --git a/statediff/publisher/ipfs/node.go b/statediff/publisher/ipfs/node.go new file mode 100644 index 000000000000..1d8daa5e89a0 --- /dev/null +++ b/statediff/publisher/ipfs/node.go @@ -0,0 +1,78 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package ipfs + +import ( + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + + "github.com/ethereum/go-ethereum/statediff/builder" +) + +type StateDiffNode struct { + *builder.StateDiff + + cid *cid.Cid + rawdata []byte +} + +func (sdn *StateDiffNode) RawData() []byte { + return sdn.rawdata +} + +func (sdn *StateDiffNode) Cid() cid.Cid { + return *sdn.cid +} + +func (sdn StateDiffNode) String() string { + return sdn.cid.String() +} + +func (sdn StateDiffNode) Loggable() map[string]interface{} { + return sdn.cid.Loggable() +} + +func (sdn StateDiffNode) Resolve(path []string) (interface{}, []string, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Tree(path string, depth int) []string { + panic("implement me") +} + +func (sdn StateDiffNode) ResolveLink(path []string) (*ipld.Link, []string, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Copy() ipld.Node { + panic("implement me") +} + +func (sdn StateDiffNode) Links() []*ipld.Link { + panic("implement me") +} + +func (sdn StateDiffNode) Stat() (*ipld.NodeStat, error) { + panic("implement me") +} + +func (sdn StateDiffNode) Size() (uint64, error) { + panic("implement me") +} \ No newline at end of file diff --git a/statediff/publisher/publisher.go b/statediff/publisher/publisher.go new file mode 100644 index 000000000000..404871405ff2 --- /dev/null +++ b/statediff/publisher/publisher.go @@ -0,0 +1,66 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package publisher + +import ( + "errors" + + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/publisher/ipfs" +) + +type Publisher interface { + PublishStateDiff(sd *builder.StateDiff) (string, error) +} + +type publisher struct { + ipfs.DagPutter + statediff.Config +} + +func NewPublisher(config statediff.Config) (*publisher, error) { + adder, err := ipfs.NewAdder(config.Path) + if err != nil { + return nil, err + } + + return &publisher{ + DagPutter: ipfs.NewDagPutter(adder), + Config: config, + }, nil +} + +func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { + switch p.Mode { + case statediff.IPLD: + cidStr, err := p.DagPut(sd) + if err != nil { + return "", err + } + + return cidStr, err + case statediff.LDB: + case statediff.SQL: + default: + } + + return "", errors.New("state diff publisher: unhandled publishing mode") +} \ No newline at end of file diff --git a/statediff/publisher/publisher_suite_test.go b/statediff/publisher/publisher_suite_test.go new file mode 100644 index 000000000000..29d10e2a105c --- /dev/null +++ b/statediff/publisher/publisher_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package publisher_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiffPublisher(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Publisher Suite") +} \ No newline at end of file diff --git a/statediff/publisher/publisher_test.go b/statediff/publisher/publisher_test.go new file mode 100644 index 000000000000..520163ddc476 --- /dev/null +++ b/statediff/publisher/publisher_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package publisher_test \ No newline at end of file diff --git a/statediff/service/service.go b/statediff/service/service.go new file mode 100644 index 000000000000..b3da7532d796 --- /dev/null +++ b/statediff/service/service.go @@ -0,0 +1,94 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package service + +import ( + "log" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/extractor" + "github.com/ethereum/go-ethereum/statediff/publisher" +) + +type StateDiffService struct { + builder builder.Builder + extractor extractor.Extractor + blockchain *core.BlockChain +} + +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*StateDiffService, error) { + config := statediff.Config{} + b := builder.NewBuilder(db) + p, err := publisher.NewPublisher(config) + if err != nil { + return nil, nil + } + + e, _ := extractor.NewExtractor(b, p) + return &StateDiffService{ + blockchain: blockChain, + extractor: e, + }, nil +} + +func (StateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} + +} + +func (StateDiffService) APIs() []rpc.API { + return []rpc.API{} +} + +func (sds *StateDiffService) loop (sub event.Subscription, events chan core.ChainHeadEvent) { + defer sub.Unsubscribe() + + for { + select { + case ev, ok := <-events: + if !ok { + log.Fatalf("Error getting chain head event from subscription.") + } + log.Println("doing something with an event", ev) + previousBlock := ev.Block + //TODO: figure out the best way to get the previous block + currentBlock := ev.Block + sds.extractor.ExtractStateDiff(*previousBlock, *currentBlock) + } + } + +} +func (sds *StateDiffService) Start(server *p2p.Server) error { + events := make(chan core.ChainHeadEvent, 10) + sub := sds.blockchain.SubscribeChainHeadEvent(events) + + go sds.loop(sub, events) + + return nil +} +func (StateDiffService) Stop() error { + return nil +} diff --git a/statediff/service/service_suite_test.go b/statediff/service/service_suite_test.go new file mode 100644 index 000000000000..60212a8fd617 --- /dev/null +++ b/statediff/service/service_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package service_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiffService(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Service Suite") +} \ No newline at end of file diff --git a/statediff/service/service_test.go b/statediff/service/service_test.go new file mode 100644 index 000000000000..99e6cfd639a3 --- /dev/null +++ b/statediff/service/service_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package service_test \ No newline at end of file diff --git a/statediff/statediff_integration_test.go b/statediff/statediff_integration_test.go new file mode 100644 index 000000000000..de5e170957b0 --- /dev/null +++ b/statediff/statediff_integration_test.go @@ -0,0 +1,20 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff_test \ No newline at end of file diff --git a/statediff/statediff_suite_test.go b/statediff/statediff_suite_test.go new file mode 100644 index 000000000000..99834db4810e --- /dev/null +++ b/statediff/statediff_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStateDiff(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "StateDiff Suite") +} diff --git a/statediff/testhelpers/mocks.go b/statediff/testhelpers/mocks.go new file mode 100644 index 000000000000..b4608e455391 --- /dev/null +++ b/statediff/testhelpers/mocks.go @@ -0,0 +1,68 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package testhelpers + +import ( + "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/builder" +) + +var MockError = errors.New("mock error") + +type MockBuilder struct { + OldStateRoot common.Hash + NewStateRoot common.Hash + BlockNumber int64 + BlockHash common.Hash + stateDiff *builder.StateDiff + builderError error +} + +func (builder *MockBuilder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*builder.StateDiff, error) { + builder.OldStateRoot = oldStateRoot + builder.NewStateRoot = newStateRoot + builder.BlockNumber = blockNumber + builder.BlockHash = blockHash + + return builder.stateDiff, builder.builderError +} + +func (builder *MockBuilder) SetStateDiffToBuild(stateDiff *builder.StateDiff) { + builder.stateDiff = stateDiff +} + +func (builder *MockBuilder) SetBuilderError(err error) { + builder.builderError = err +} + +type MockPublisher struct{ + StateDiff *builder.StateDiff + publisherError error +} + +func (publisher *MockPublisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { + publisher.StateDiff = sd + return "", publisher.publisherError +} + +func (publisher *MockPublisher) SetPublisherError(err error) { + publisher.publisherError = err +}