diff --git a/cl/cltypes/types_encoding.go b/cl/cltypes/types_encoding.go index aa7fb5e4489..4bbf0574114 100644 --- a/cl/cltypes/types_encoding.go +++ b/cl/cltypes/types_encoding.go @@ -378,6 +378,7 @@ func (s *SyncAggregate) HashTreeRootWith(hh *ssz.Hasher) (err error) { return } + // MarshalSSZ ssz marshals the BeaconBodyBellatrix object func (b *BeaconBodyBellatrix) MarshalSSZ() ([]byte, error) { return ssz.MarshalSSZ(b) diff --git a/cmd/devnet/commands/all.go b/cmd/devnet/commands/all.go index 34fa379be69..e95c7f5f268 100644 --- a/cmd/devnet/commands/all.go +++ b/cmd/devnet/commands/all.go @@ -2,6 +2,7 @@ package commands import ( "fmt" + "github.com/ledgerwatch/erigon/cmd/devnet/models" "github.com/ledgerwatch/erigon/cmd/devnet/services" ) diff --git a/cmd/devnet/commands/block.go b/cmd/devnet/commands/block.go index 1e505fc1f81..8e1444fa8d0 100644 --- a/cmd/devnet/commands/block.go +++ b/cmd/devnet/commands/block.go @@ -2,6 +2,7 @@ package commands import ( "fmt" + "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" "github.com/ledgerwatch/erigon/cmd/devnet/models" "github.com/ledgerwatch/erigon/cmd/devnet/requests" diff --git a/cmd/devnet/devnetutils/utils.go b/cmd/devnet/devnetutils/utils.go index 8c6571adb44..98a1f94056d 100644 --- a/cmd/devnet/devnetutils/utils.go +++ b/cmd/devnet/devnetutils/utils.go @@ -3,13 +3,14 @@ package devnetutils import ( "encoding/json" "fmt" + "os/exec" + "strconv" + "strings" + "github.com/ledgerwatch/erigon/cmd/rpctest/rpctest" "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/crypto" - "os/exec" - "strconv" - "strings" "github.com/ledgerwatch/erigon/cmd/devnet/models" ) diff --git a/cmd/devnet/main.go b/cmd/devnet/main.go index d531b20b959..90cedff88b1 100644 --- a/cmd/devnet/main.go +++ b/cmd/devnet/main.go @@ -1,12 +1,12 @@ package main import ( - "github.com/ledgerwatch/erigon/cmd/devnet/models" "sync" "time" "github.com/ledgerwatch/erigon/cmd/devnet/commands" "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" + "github.com/ledgerwatch/erigon/cmd/devnet/models" "github.com/ledgerwatch/erigon/cmd/devnet/node" "github.com/ledgerwatch/erigon/cmd/devnet/services" ) diff --git a/cmd/rpcdaemon/commands/debug_api.go b/cmd/rpcdaemon/commands/debug_api.go index da998487a71..8fa30ccb8ca 100644 --- a/cmd/rpcdaemon/commands/debug_api.go +++ b/cmd/rpcdaemon/commands/debug_api.go @@ -50,7 +50,7 @@ func NewPrivateDebugAPI(base *BaseAPI, db kv.RoDB, gascap uint64) *PrivateDebugA } } -// StorageRangeAt implements debug_storageRangeAt. Returns information about a range of storage locations (if any) for the given address. +// storageRangeAt implements debug_storageRangeAt. Returns information about a range of storage locations (if any) for the given address. func (api *PrivateDebugAPIImpl) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex uint64, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { tx, err := api.db.BeginRo(ctx) if err != nil { @@ -79,7 +79,7 @@ func (api *PrivateDebugAPIImpl) StorageRangeAt(ctx context.Context, blockHash co if err != nil { return StorageRangeResult{}, err } - return StorageRangeAt(stateReader.(*state.PlainState), contractAddress, keyStart, maxResult) + return storageRangeAt(stateReader.(*state.PlainState), contractAddress, keyStart, maxResult) } // AccountRange implements debug_accountRange. Returns a range of accounts involved in the given block rangeb diff --git a/cmd/rpcdaemon/commands/debug_api_test.go b/cmd/rpcdaemon/commands/debug_api_test.go index aa2f7e7052e..e850f2d8419 100644 --- a/cmd/rpcdaemon/commands/debug_api_test.go +++ b/cmd/rpcdaemon/commands/debug_api_test.go @@ -192,3 +192,69 @@ func TestTraceTransactionNoRefund(t *testing.T) { } } } + +/* +func TestStorageRangeAt(t *testing.T) { + t.Parallel() + + // Create a state where account 0x010000... has a few storage entries. + var ( + //state, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + addr = common.Address{0x01} + keys = []common.Hash{ // hashes of Keys of storage + common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"), + common.HexToHash("426fcb404ab2d5d8e61a3d918108006bbb0a9be65e92235bb10eefbdb6dcd053"), + common.HexToHash("48078cfed56339ea54962e72c37c7f588fc4f8e5bc173827ba75cb10a63a96a5"), + common.HexToHash("5723d2c3a83af9b735e3b7f21531e5623d183a9095a56604ead41f3582fdfb75"), + } + storage = storageMap{ + keys[0]: {Key: &common.Hash{0x02}, Value: common.Hash{0x01}}, + keys[1]: {Key: &common.Hash{0x04}, Value: common.Hash{0x02}}, + keys[2]: {Key: &common.Hash{0x01}, Value: common.Hash{0x03}}, + keys[3]: {Key: &common.Hash{0x03}, Value: common.Hash{0x04}}, + } + ) + //for _, entry := range storage { + // state.SetState(addr, *entry.Key, entry.Value) + //} + + // Check a few combinations of limit and start/end. + tests := []struct { + start []byte + limit int + want StorageRangeResult + }{ + { + start: []byte{}, limit: 0, + want: StorageRangeResult{storageMap{}, &keys[0]}, + }, + { + start: []byte{}, limit: 100, + want: StorageRangeResult{storage, nil}, + }, + { + start: []byte{}, limit: 2, + want: StorageRangeResult{storageMap{keys[0]: storage[keys[0]], keys[1]: storage[keys[1]]}, &keys[2]}, + }, + { + start: []byte{0x00}, limit: 4, + want: StorageRangeResult{storage, nil}, + }, + { + start: []byte{0x40}, limit: 2, + want: StorageRangeResult{storageMap{keys[1]: storage[keys[1]], keys[2]: storage[keys[2]]}, &keys[3]}, + }, + } + for _, test := range tests { + //tr, err := state.StorageTrie(addr) + //if err != nil { + // t.Error(err) + //} + result, err := storageRangeAt(stateReader.(*state.PlainState), test.start, test.limit) + if err != nil { + t.Error(err) + } + require.EqualValues(t, test.want, result) + } +} +*/ diff --git a/cmd/rpcdaemon/commands/erigon_receipts_test.go b/cmd/rpcdaemon/commands/erigon_receipts_test.go index 01903082967..17d853bbc82 100644 --- a/cmd/rpcdaemon/commands/erigon_receipts_test.go +++ b/cmd/rpcdaemon/commands/erigon_receipts_test.go @@ -2,6 +2,7 @@ package commands import ( "context" + "encoding/json" "math/big" "testing" @@ -24,6 +25,54 @@ import ( "github.com/stretchr/testify/require" ) +func TestGetLogs(t *testing.T) { + assert := assert.New(t) + m, _, _ := rpcdaemontest.CreateTestSentry(t) + br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots) + agg := m.HistoryV3Components() + baseApi := NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + { + ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000) + + logs, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)}) + assert.NoError(err) + assert.Equal(uint64(10), logs[0].BlockNumber) + + //filer by wrong address + logs, err = ethApi.GetLogs(context.Background(), filters.FilterCriteria{ + Addresses: common.Addresses{common.Address{}}, + }) + assert.NoError(err) + assert.Equal(0, len(logs)) + + //filer by wrong address + logs, err = ethApi.GetLogs(m.Ctx, filters.FilterCriteria{ + Topics: [][]common.Hash{{common.HexToHash("0x68f6a0f063c25c6678c443b9a484086f15ba8f91f60218695d32a5251f2050eb")}}, + }) + assert.NoError(err) + assert.Equal(1, len(logs)) + } + // + //api := NewErigonAPI(baseApi, m.DB, nil) + //logs, err := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)}) + //assert.NoError(err) + //assert.Equal(uint64(10), logs[0].BlockNumber) + // + ////filer by wrong address + //logs, err = api.GetLogs(m.Ctx, filters.FilterCriteria{ + // Addresses: common.Addresses{common.Address{}}, + //}) + //assert.NoError(err) + //assert.Equal(0, len(logs)) + // + ////filer by wrong address + //logs, err = api.GetLogs(m.Ctx, filters.FilterCriteria{ + // Topics: [][]common.Hash{{common.HexToHash("0x68f6a0f063c25c6678c443b9a484086f15ba8f91f60218695d32a5251f2050eb")}}, + //}) + //assert.NoError(err) + //assert.Equal(1, len(logs)) +} + func TestErigonGetLatestLogs(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) @@ -32,7 +81,7 @@ func TestErigonGetLatestLogs(t *testing.T) { db := m.DB agg := m.HistoryV3Components() api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), db, nil) - expectedLogs, _ := api.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) + expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) expectedErigonLogs := make([]*types.ErigonLog, 0) for i := len(expectedLogs) - 1; i >= 0; i-- { @@ -49,8 +98,8 @@ func TestErigonGetLatestLogs(t *testing.T) { Timestamp: expectedLogs[i].Timestamp, }) } - actual, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{}, filters.LogFilterOptions{ - LogCount: uint64((len(expectedLogs))), + actual, err := api.GetLatestLogs(m.Ctx, filters.FilterCriteria{}, filters.LogFilterOptions{ + LogCount: uint64(len(expectedLogs)), }) if err != nil { t.Errorf("calling erigon_getLatestLogs: %v", err) @@ -67,7 +116,7 @@ func TestErigonGetLatestLogsIgnoreTopics(t *testing.T) { db := m.DB agg := m.HistoryV3Components() api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), db, nil) - expectedLogs, _ := api.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) + expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) expectedErigonLogs := make([]*types.ErigonLog, 0) for i := len(expectedLogs) - 1; i >= 0; i-- { @@ -97,7 +146,7 @@ func TestErigonGetLatestLogsIgnoreTopics(t *testing.T) { expectedLogs[i].Topics[0], }) } - actual, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{Topics: containsTopics}, filters.LogFilterOptions{ + actual, err := api.GetLatestLogs(m.Ctx, filters.FilterCriteria{Topics: containsTopics}, filters.LogFilterOptions{ BlockCount: blockCount, }) if err != nil { @@ -158,33 +207,24 @@ func TestGetBlockReceiptsByBlockHash(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil) - if m.HistoryV3 { - t.Skip("ErigonV3 doesn't store receipts in DB. Can't use \"rawdb\" package in this case. need somehow change assertion") + expect := map[uint64]string{ + 0: `[]`, + 1: `[{"blockHash":"0x63b978611c906a61e4a8333fedeea8d62a1c869fc9a19acf6ed0cc5139247eda","blockNumber":"0x1","contractAddress":null,"cumulativeGasUsed":"0x5208","effectiveGasPrice":"0x0","from":"0x71562b71999873db5b286df957af199ec94617f7","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":"0x703c4b2bd70c169f5717101caee543299fc946c7","transactionHash":"0x9ca7a9e6bf23353fc5ac37f5c5676db1accec4af83477ac64cdcaa37f3a837f9","transactionIndex":"0x0","type":"0x0"}]`, + 2: `[{"blockHash":"0xd3294fcc342ff74be4ae07fb25cd3b2fbb6c2b7830f212ee0723da956e70e099","blockNumber":"0x2","contractAddress":null,"cumulativeGasUsed":"0x5208","effectiveGasPrice":"0x0","from":"0x71562b71999873db5b286df957af199ec94617f7","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":"0x703c4b2bd70c169f5717101caee543299fc946c7","transactionHash":"0xf190eed1578cdcfe69badd05b7ef183397f336dc3de37baa4adbfb4bc657c11e","transactionIndex":"0x0","type":"0x0"},{"blockHash":"0xd3294fcc342ff74be4ae07fb25cd3b2fbb6c2b7830f212ee0723da956e70e099","blockNumber":"0x2","contractAddress":null,"cumulativeGasUsed":"0xa410","effectiveGasPrice":"0x0","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":"0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e","transactionHash":"0x309a030e44058e435a2b01302006880953e2c9319009db97013eb130d7a24eab","transactionIndex":"0x1","type":"0x0"}]`, + 3: `[]`, + 4: `[]`, } err := m.DB.View(m.Ctx, func(tx kv.Tx) error { for i := uint64(0); i <= rawdb.ReadCurrentHeader(tx).Number.Uint64(); i++ { - block, err := rawdb.ReadBlockByNumber(tx, i) - if err != nil { - return err - } - - r, err := rawdb.ReadReceiptsByHash(tx, block.Hash()) - if err != nil { - return err - } - - marshaledReceipts := make([]map[string]interface{}, 0, len(r)) - for _, receipt := range r { - txn := block.Transactions()[receipt.TransactionIndex] - marshaledReceipts = append(marshaledReceipts, marshalReceipt(receipt, txn, m.ChainConfig, block, txn.Hash(), true)) - } + block := rawdb.ReadHeaderByNumber(tx, i) receiptsFromBlock, err := api.GetBlockReceiptsByBlockHash(context.Background(), block.Hash()) if err != nil { return err } - assert.EqualValues(t, marshaledReceipts, receiptsFromBlock) + a, _ := json.Marshal(receiptsFromBlock) + assert.Equal(t, expect[block.Number.Uint64()], string(a)) } return nil }) diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index 64e7fab72a1..dffdbf390bc 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -311,7 +311,6 @@ func applyFiltersV3(out *roaring64.Bitmap, tx kv.TemporalTx, begin, end uint64, } toTxNum++ - out.Clear() out.AddRange(fromTxNum, toTxNum) // [from,to) topicsBitmap, err := getTopicsBitmapV3(tx, crit.Topics, fromTxNum, toTxNum) if err != nil { @@ -484,29 +483,29 @@ func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]common.Hash, from, to uint64) (*roaring64.Bitmap, error) { var result *roaring64.Bitmap for _, sub := range topics { - var bitmapForORing roaring64.Bitmap + bitmapForORing := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bitmapForORing) + for _, topic := range sub { it, err := tx.IndexRange(temporal.LogTopicIdx, topic.Bytes(), from, to) if err != nil { return nil, err } - for it.HasNext() { - n, err := it.NextBatch() - if err != nil { - return nil, err - } - bitmapForORing.AddMany(n) + bm, err := it.ToBitmap() + if err != nil { + return nil, err } + bitmapForORing.Or(bm) } if bitmapForORing.GetCardinality() == 0 { continue } if result == nil { - result = &bitmapForORing + result = bitmapForORing.Clone() continue } - result = roaring64.And(&bitmapForORing, result) + result = roaring64.And(bitmapForORing, result) } return result, nil } @@ -525,15 +524,10 @@ func getAddrsBitmapV3(tx kv.TemporalTx, addrs []common.Address, from, to uint64) if err != nil { return nil, err } - m := roaring64.New() - for it.HasNext() { - n, err := it.NextBatch() - if err != nil { - return nil, err - } - m.AddMany(n) + rx[idx], err = it.ToBitmap() + if err != nil { + return nil, err } - rx[idx] = m } return roaring64.FastOr(rx...), nil } diff --git a/cmd/rpcdaemon/commands/storage_range.go b/cmd/rpcdaemon/commands/storage_range.go index 9e421364e06..726cfecfb13 100644 --- a/cmd/rpcdaemon/commands/storage_range.go +++ b/cmd/rpcdaemon/commands/storage_range.go @@ -9,12 +9,12 @@ import ( // StorageRangeResult is the result of a debug_storageRangeAt API call. type StorageRangeResult struct { - Storage StorageMap `json:"storage"` + Storage storageMap `json:"storage"` NextKey *common.Hash `json:"nextKey"` // nil if Storage includes the last key in the trie. } -// StorageMap a map from storage locations to StorageEntry items -type StorageMap map[common.Hash]StorageEntry +// storageMap a map from storage locations to StorageEntry items +type storageMap map[common.Hash]StorageEntry // StorageEntry an entry in storage of the account type StorageEntry struct { @@ -26,8 +26,8 @@ type walker interface { ForEachStorage(addr common.Address, startLocation common.Hash, cb func(key, seckey common.Hash, value uint256.Int) bool, maxResults int) error } -func StorageRangeAt(stateReader walker, contractAddress common.Address, start []byte, maxResult int) (StorageRangeResult, error) { - result := StorageRangeResult{Storage: StorageMap{}} +func storageRangeAt(stateReader walker, contractAddress common.Address, start []byte, maxResult int) (StorageRangeResult, error) { + result := StorageRangeResult{Storage: storageMap{}} resultCount := 0 if err := stateReader.ForEachStorage(contractAddress, common.BytesToHash(start), func(key, seckey common.Hash, value uint256.Int) bool { diff --git a/cmd/rpcdaemon/commands/trace_filtering.go b/cmd/rpcdaemon/commands/trace_filtering.go index 3de407dcf95..bfcab17687e 100644 --- a/cmd/rpcdaemon/commands/trace_filtering.go +++ b/cmd/rpcdaemon/commands/trace_filtering.go @@ -233,36 +233,68 @@ func (api *TraceAPIImpl) Block(ctx context.Context, blockNr rpc.BlockNumber) (Pa return out, err } -func traceFilterBitmaps(tx kv.Tx, req TraceFilterRequest, fromBlock, toBlock uint64) (fromAddresses, toAddresses map[common.Address]struct{}, allBlocks *roaring64.Bitmap, err error) { +func traceFilterBitmaps(tx kv.Tx, req TraceFilterRequest, from, to uint64) (fromAddresses, toAddresses map[common.Address]struct{}, allBlocks *roaring64.Bitmap, err error) { fromAddresses = make(map[common.Address]struct{}, len(req.FromAddress)) toAddresses = make(map[common.Address]struct{}, len(req.ToAddress)) allBlocks = roaring64.New() var blocksTo roaring64.Bitmap - for _, addr := range req.FromAddress { - if addr != nil { - b, err := bitmapdb.Get64(tx, kv.CallFromIndex, addr.Bytes(), fromBlock, toBlock) - if err != nil { + if ttx, casted := tx.(kv.TemporalTx); casted { + for _, addr := range req.FromAddress { + if addr != nil { + it, err := ttx.IndexRange(temporal.TracesFromIdx, addr.Bytes(), from, to) if errors.Is(err, ethdb.ErrKeyNotFound) { continue } - return nil, nil, nil, err + b, err := it.ToBitmap() + if err != nil { + return nil, nil, nil, err + } + allBlocks.Or(b) + fromAddresses[*addr] = struct{}{} } - allBlocks.Or(b) - fromAddresses[*addr] = struct{}{} } - } - for _, addr := range req.ToAddress { - if addr != nil { - b, err := bitmapdb.Get64(tx, kv.CallToIndex, addr.Bytes(), fromBlock, toBlock) - if err != nil { + for _, addr := range req.ToAddress { + if addr != nil { + it, err := ttx.IndexRange(temporal.TracesToIdx, addr.Bytes(), from, to) if errors.Is(err, ethdb.ErrKeyNotFound) { continue } - return nil, nil, nil, err + b, err := it.ToBitmap() + if err != nil { + return nil, nil, nil, err + } + blocksTo.Or(b) + toAddresses[*addr] = struct{}{} + } + } + } else { + for _, addr := range req.FromAddress { + if addr != nil { + b, err := bitmapdb.Get64(tx, kv.CallFromIndex, addr.Bytes(), from, to) + if err != nil { + if errors.Is(err, ethdb.ErrKeyNotFound) { + continue + } + return nil, nil, nil, err + } + allBlocks.Or(b) + fromAddresses[*addr] = struct{}{} + } + } + + for _, addr := range req.ToAddress { + if addr != nil { + b, err := bitmapdb.Get64(tx, kv.CallToIndex, addr.Bytes(), from, to) + if err != nil { + if errors.Is(err, ethdb.ErrKeyNotFound) { + continue + } + return nil, nil, nil, err + } + blocksTo.Or(b) + toAddresses[*addr] = struct{}{} } - blocksTo.Or(b) - toAddresses[*addr] = struct{}{} } } @@ -277,10 +309,10 @@ func traceFilterBitmaps(tx kv.Tx, req TraceFilterRequest, fromBlock, toBlock uin // Special case - if no addresses specified, take all traces if len(req.FromAddress) == 0 && len(req.ToAddress) == 0 { - allBlocks.AddRange(fromBlock, toBlock) + allBlocks.AddRange(from, to) } else { - allBlocks.RemoveRange(0, fromBlock) - allBlocks.RemoveRange(toBlock, uint64(0x100000000)) + allBlocks.RemoveRange(0, from) + allBlocks.RemoveRange(to, uint64(0x100000000)) } return fromAddresses, toAddresses, allBlocks, nil @@ -544,64 +576,9 @@ func (api *TraceAPIImpl) filterV3(ctx context.Context, dbtx kv.TemporalTx, fromB return err } toTxNum++ //+1 because internally Erigon using semantic [from, to), but some RPC have different semantic - - fromAddresses := make(map[common.Address]struct{}, len(req.FromAddress)) - toAddresses := make(map[common.Address]struct{}, len(req.ToAddress)) - - var ( - allTxs roaring64.Bitmap - txsTo roaring64.Bitmap - ) - - for _, addr := range req.FromAddress { - if addr != nil { - it, err := dbtx.IndexRange(temporal.TracesFromIdx, addr.Bytes(), fromTxNum, toTxNum) - if err != nil { - return err - } - for it.HasNext() { - n, err := it.NextBatch() - if err != nil { - return err - } - allTxs.AddMany(n) - } - fromAddresses[*addr] = struct{}{} - } - } - - for _, addr := range req.ToAddress { - if addr != nil { - it, err := dbtx.IndexRange(temporal.TracesToIdx, addr.Bytes(), fromTxNum, toTxNum) - if err != nil { - return err - } - for it.HasNext() { - n, err := it.NextBatch() - if err != nil { - return err - } - txsTo.AddMany(n) - } - toAddresses[*addr] = struct{}{} - } - } - - switch req.Mode { - case TraceFilterModeIntersection: - allTxs.And(&txsTo) - case TraceFilterModeUnion: - fallthrough - default: - allTxs.Or(&txsTo) - } - - // Special case - if no addresses specified, take all traces - if len(req.FromAddress) == 0 && len(req.ToAddress) == 0 { - allTxs.AddRange(fromTxNum, toTxNum+1) - } else { - allTxs.RemoveRange(0, fromTxNum) - allTxs.RemoveRange(toTxNum, uint64(0x1000000000000)) + fromAddresses, toAddresses, allTxs, err := traceFilterBitmaps(dbtx, req, fromTxNum, toTxNum) + if err != nil { + return err } chainConfig, err := api.chainConfig(dbtx) diff --git a/core/state/history_reader_v3.go b/core/state/history_reader_v3.go index b19d3f9088d..ee31bacd4fa 100644 --- a/core/state/history_reader_v3.go +++ b/core/state/history_reader_v3.go @@ -7,7 +7,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv" libstate "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/common" - "github.com/ledgerwatch/erigon/common/dbutils" "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types/accounts" ) @@ -17,7 +16,6 @@ type HistoryReaderV3 struct { ac *libstate.Aggregator22Context txNum uint64 trace bool - tx kv.Tx ttx kv.TemporalTx } @@ -31,8 +29,9 @@ func (hr *HistoryReaderV3) SetAc(ac *libstate.Aggregator22Context) { func (hr *HistoryReaderV3) SetTx(tx kv.Tx) { if ttx, casted := tx.(kv.TemporalTx); casted { hr.ttx = ttx + } else { + panic("why") } - hr.tx = tx } func (hr *HistoryReaderV3) SetTxNum(txNum uint64) { hr.txNum = txNum } func (hr *HistoryReaderV3) SetTrace(trace bool) { hr.trace = trace } @@ -41,59 +40,17 @@ func (hr *HistoryReaderV3) ReadAccountData(address common.Address) (*accounts.Ac var enc []byte var ok bool var err error - if hr.ttx != nil { - enc, ok, err = hr.ttx.DomainGet(temporal.AccountsDomain, address.Bytes(), nil, hr.txNum) - if err != nil || !ok || len(enc) == 0 { - if hr.trace { - fmt.Printf("ReadAccountData [%x] => []\n", address) - } - return nil, err - } - var a accounts.Account - if err := a.DecodeForStorage(enc); err != nil { - return nil, fmt.Errorf("ReadAccountData(%x): %w", address, err) - } - if hr.trace { - fmt.Printf("ReadAccountData [%x] => [nonce: %d, balance: %d, codeHash: %x]\n", address, a.Nonce, &a.Balance, a.CodeHash) - } - return &a, nil - } - enc, ok, err = hr.ac.ReadAccountDataNoStateWithRecent(address.Bytes(), hr.txNum) - if err != nil { - return nil, err - } - if ok { - if len(enc) == 0 { - if hr.trace { - fmt.Printf("ReadAccountData [%x] => []\n", address) - } - return nil, nil - } - var a accounts.Account - if err := accounts.DeserialiseV3(&a, enc); err != nil { - return nil, fmt.Errorf("ReadAccountData(%x): %w", address, err) - } - if hr.trace { - fmt.Printf("ReadAccountData [%x] => [nonce: %d, balance: %d, codeHash: %x]\n", address, a.Nonce, &a.Balance, a.CodeHash) - } - return &a, nil - } - enc, err = hr.tx.GetOne(kv.PlainState, address.Bytes()) - if err != nil { - return nil, err - } - - if len(enc) == 0 { + enc, ok, err = hr.ttx.DomainGet(temporal.AccountsDomain, address.Bytes(), nil, hr.txNum) + if err != nil || !ok || len(enc) == 0 { if hr.trace { fmt.Printf("ReadAccountData [%x] => []\n", address) } - return nil, nil + return nil, err } var a accounts.Account if err := a.DecodeForStorage(enc); err != nil { return nil, fmt.Errorf("ReadAccountData(%x): %w", address, err) } - if hr.trace { fmt.Printf("ReadAccountData [%x] => [nonce: %d, balance: %d, codeHash: %x]\n", address, a.Nonce, &a.Balance, a.CodeHash) } @@ -101,84 +58,21 @@ func (hr *HistoryReaderV3) ReadAccountData(address common.Address) (*accounts.Ac } func (hr *HistoryReaderV3) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { - if hr.ttx != nil { - enc, _, err := hr.ttx.DomainGet(temporal.StorageDomain, append(address.Bytes(), common2.EncodeTs(incarnation)...), key.Bytes(), hr.txNum) - return enc, err - } - - enc, ok, err := hr.ac.ReadAccountStorageNoStateWithRecent(address.Bytes(), key.Bytes(), hr.txNum) - if err != nil { - return nil, err - } - if !ok { - k := dbutils.PlainGenerateCompositeStorageKey(address[:], incarnation, key.Bytes()) - enc, err = hr.tx.GetOne(kv.PlainState, k) - if err != nil { - return nil, err - } - } - if hr.trace { - if enc == nil { - fmt.Printf("ReadAccountStorage [%x] [%x] => []\n", address, key.Bytes()) - } else { - fmt.Printf("ReadAccountStorage [%x] [%x] => [%x]\n", address, key.Bytes(), enc) - } - } - if enc == nil { - return nil, nil - } - return enc, nil + enc, _, err := hr.ttx.DomainGet(temporal.StorageDomain, append(address.Bytes(), common2.EncodeTs(incarnation)...), key.Bytes(), hr.txNum) + return enc, err } func (hr *HistoryReaderV3) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { if codeHash == emptyCodeHashH { return nil, nil } - if hr.ttx != nil { - enc, _, err := hr.ttx.DomainGet(temporal.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) - return enc, err - } - - enc, ok, err := hr.ac.ReadAccountCodeNoStateWithRecent(address.Bytes(), hr.txNum) - if err != nil { - return nil, err - } - if !ok { - enc, err = hr.tx.GetOne(kv.Code, codeHash[:]) - if err != nil { - return nil, err - } - } - if hr.trace { - fmt.Printf("ReadAccountCode [%x %x] => [%x]\n", address, codeHash, enc) - } - return enc, nil + enc, _, err := hr.ttx.DomainGet(temporal.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) + return enc, err } func (hr *HistoryReaderV3) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { - if hr.ttx != nil { - enc, _, err := hr.ttx.DomainGet(temporal.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) - return len(enc), err - } - enc, ok, err := hr.ac.ReadAccountCodeNoStateWithRecent(address.Bytes(), hr.txNum) - size := len(enc) - if err != nil { - return 0, err - } - if !ok { - enc, err := hr.tx.GetOne(kv.Code, codeHash[:]) - if err != nil { - return 0, err - } - size = len(enc) - } - if err != nil { - return 0, err - } - if hr.trace { - fmt.Printf("ReadAccountCodeSize [%x] => [%d]\n", address, size) - } - return size, nil + enc, _, err := hr.ttx.DomainGet(temporal.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) + return len(enc), err } func (hr *HistoryReaderV3) ReadAccountIncarnation(address common.Address) (uint64, error) { diff --git a/core/state/temporal/kv_temporal.go b/core/state/temporal/kv_temporal.go index 41fd714aaeb..d92bac58567 100644 --- a/core/state/temporal/kv_temporal.go +++ b/core/state/temporal/kv_temporal.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/RoaringBitmap/roaring" "github.com/RoaringBitmap/roaring/roaring64" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" @@ -247,7 +246,7 @@ type Cursor struct { } // [fromTs, toTs) -func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64) (timestamps kv.UnaryStream[uint64], err error) { +func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64) (timestamps kv.U64Stream, err error) { if tx.hitoryV3 { switch name { case LogTopicIdx: @@ -277,15 +276,15 @@ func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64) ( if err != nil { return nil, err } - bm = castBitmapTo64(bm32) + bm = bitmapdb.CastBitmapTo64(bm32) case LogAddrIdx: bm32, err := bitmapdb.Get(tx, kv.LogAddressIndex, key, uint32(fromTs), uint32(toTs)) if err != nil { return nil, err } - bm = castBitmapTo64(bm32) + bm = bitmapdb.CastBitmapTo64(bm32) case TracesFromIdx: - bm, err = bitmapdb.Get64(tx, kv.TracesFromIdx, key, fromTs, toTs) + bm, err = bitmapdb.Get64(tx, kv.CallFromIndex, key, fromTs, toTs) if err != nil { return nil, err } @@ -297,14 +296,6 @@ func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64) ( default: return nil, fmt.Errorf("unexpected history name: %s", name) } - return kv.StreamArray(bm.ToArray()), nil + return bitmapdb.NewBitmapStream(bm), nil } } - -func castBitmapTo64(in *roaring.Bitmap) *roaring64.Bitmap { - bm := roaring64.New() - for _, v := range in.ToArray() { - bm.Add(uint64(v)) - } - return bm -} diff --git a/go.mod b/go.mod index 6b053aa4b25..a9b4799c813 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230112022639-db1aa7296e68 + github.com/ledgerwatch/erigon-lib v0.0.0-20230112023724-cc22f25b0990 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230106211435-2670b273bb55 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 93b11df24cd..fa280f74ad6 100644 --- a/go.sum +++ b/go.sum @@ -565,8 +565,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230112022639-db1aa7296e68 h1:2Vmzyz1eEkykObQhG8uOfWcmsVnkJ/rvAAkaj8t0Xec= -github.com/ledgerwatch/erigon-lib v0.0.0-20230112022639-db1aa7296e68/go.mod h1:vFTw08qI/7NHL3z4/vBlIXRylFgZlMbnTehf7WDbcJQ= +github.com/ledgerwatch/erigon-lib v0.0.0-20230112023724-cc22f25b0990 h1:it6cqN87wBGrsir0ppWNmvz1feBE8h2ouKmj8fmNqGU= +github.com/ledgerwatch/erigon-lib v0.0.0-20230112023724-cc22f25b0990/go.mod h1:vFTw08qI/7NHL3z4/vBlIXRylFgZlMbnTehf7WDbcJQ= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230106211435-2670b273bb55 h1:4gY6eRXQiX0Te1oKTpsxj6MjXksQSSTHWiSh/j89sEY= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230106211435-2670b273bb55/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og=