e3: stream.ToBitamp() (#6562)

cmd/devnet/commands/all.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"fmt"
+
 	"github.com/ledgerwatch/erigon/cmd/devnet/models"
 	"github.com/ledgerwatch/erigon/cmd/devnet/services"
 )

cmd/devnet/commands/block.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"fmt"
+
 	"github.com/ledgerwatch/erigon/cmd/devnet/devnetutils"
 	"github.com/ledgerwatch/erigon/cmd/devnet/models"
 	"github.com/ledgerwatch/erigon/cmd/devnet/requests"

cmd/devnet/devnetutils/utils.go

@@ -3,13 +3,14 @@ package devnetutils
 import (
 	"encoding/json"
 	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+
 	"github.com/ledgerwatch/erigon/cmd/rpctest/rpctest"
 	"github.com/ledgerwatch/erigon/common"
 	"github.com/ledgerwatch/erigon/common/hexutil"
 	"github.com/ledgerwatch/erigon/crypto"
-	"os/exec"
-	"strconv"
-	"strings"
 
 	"github.com/ledgerwatch/erigon/cmd/devnet/models"
 )

cmd/devnet/main.go

@@ -1,12 +1,12 @@
 package main
 
 import (
-	"github.com/ledgerwatch/erigon/cmd/devnet/models"
 	"sync"
 	"time"
 
 	"github.com/ledgerwatch/erigon/cmd/devnet/commands"
 	"github.com/ledgerwatch/erigon/cmd/devnet/devnetutils"
+	"github.com/ledgerwatch/erigon/cmd/devnet/models"
 	"github.com/ledgerwatch/erigon/cmd/devnet/node"
 	"github.com/ledgerwatch/erigon/cmd/devnet/services"
 )

cmd/rpcdaemon/commands/debug_api.go

@@ -50,7 +50,7 @@ func NewPrivateDebugAPI(base *BaseAPI, db kv.RoDB, gascap uint64) *PrivateDebugA
 	}
 }
 
-// StorageRangeAt implements debug_storageRangeAt. Returns information about a range of storage locations (if any) for the given address.
+// storageRangeAt implements debug_storageRangeAt. Returns information about a range of storage locations (if any) for the given address.
 func (api *PrivateDebugAPIImpl) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex uint64, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
 	tx, err := api.db.BeginRo(ctx)
 	if err != nil {
@@ -79,7 +79,7 @@ func (api *PrivateDebugAPIImpl) StorageRangeAt(ctx context.Context, blockHash co
 	if err != nil {
 		return StorageRangeResult{}, err
 	}
-	return StorageRangeAt(stateReader.(*state.PlainState), contractAddress, keyStart, maxResult)
+	return storageRangeAt(stateReader.(*state.PlainState), contractAddress, keyStart, maxResult)
 }
 
 // AccountRange implements debug_accountRange. Returns a range of accounts involved in the given block rangeb

cmd/rpcdaemon/commands/debug_api_test.go

@@ -192,3 +192,69 @@ func TestTraceTransactionNoRefund(t *testing.T) {
 		}
 	}
 }
+
+/*
+func TestStorageRangeAt(t *testing.T) {
+	t.Parallel()
+
+	// Create a state where account 0x010000... has a few storage entries.
+	var (
+		//state, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
+		addr = common.Address{0x01}
+		keys = []common.Hash{ // hashes of Keys of storage
+			common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"),
+			common.HexToHash("426fcb404ab2d5d8e61a3d918108006bbb0a9be65e92235bb10eefbdb6dcd053"),
+			common.HexToHash("48078cfed56339ea54962e72c37c7f588fc4f8e5bc173827ba75cb10a63a96a5"),
+			common.HexToHash("5723d2c3a83af9b735e3b7f21531e5623d183a9095a56604ead41f3582fdfb75"),
+		}
+		storage = storageMap{
+			keys[0]: {Key: &common.Hash{0x02}, Value: common.Hash{0x01}},
+			keys[1]: {Key: &common.Hash{0x04}, Value: common.Hash{0x02}},
+			keys[2]: {Key: &common.Hash{0x01}, Value: common.Hash{0x03}},
+			keys[3]: {Key: &common.Hash{0x03}, Value: common.Hash{0x04}},
+		}
+	)
+	//for _, entry := range storage {
+	//	state.SetState(addr, *entry.Key, entry.Value)
+	//}
+
+	// Check a few combinations of limit and start/end.
+	tests := []struct {
+		start []byte
+		limit int
+		want  StorageRangeResult
+	}{
+		{
+			start: []byte{}, limit: 0,
+			want: StorageRangeResult{storageMap{}, &keys[0]},
+		},
+		{
+			start: []byte{}, limit: 100,
+			want: StorageRangeResult{storage, nil},
+		},
+		{
+			start: []byte{}, limit: 2,
+			want: StorageRangeResult{storageMap{keys[0]: storage[keys[0]], keys[1]: storage[keys[1]]}, &keys[2]},
+		},
+		{
+			start: []byte{0x00}, limit: 4,
+			want: StorageRangeResult{storage, nil},
+		},
+		{
+			start: []byte{0x40}, limit: 2,
+			want: StorageRangeResult{storageMap{keys[1]: storage[keys[1]], keys[2]: storage[keys[2]]}, &keys[3]},
+		},
+	}
+	for _, test := range tests {
+		//tr, err := state.StorageTrie(addr)
+		//if err != nil {
+		//	t.Error(err)
+		//}
+		result, err := storageRangeAt(stateReader.(*state.PlainState), test.start, test.limit)
+		if err != nil {
+			t.Error(err)
+		}
+		require.EqualValues(t, test.want, result)
+	}
+}
+*/

cmd/rpcdaemon/commands/erigon_receipts_test.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"context"
+	"encoding/json"
 	"math/big"
 	"testing"
 
@@ -24,6 +25,54 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestGetLogs(t *testing.T) {
+	assert := assert.New(t)
+	m, _, _ := rpcdaemontest.CreateTestSentry(t)
+	br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots)
+	agg := m.HistoryV3Components()
+	baseApi := NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine)
+	{
+		ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000)
+
+		logs, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)})
+		assert.NoError(err)
+		assert.Equal(uint64(10), logs[0].BlockNumber)
+
+		//filer by wrong address
+		logs, err = ethApi.GetLogs(context.Background(), filters.FilterCriteria{
+			Addresses: common.Addresses{common.Address{}},
+		})
+		assert.NoError(err)
+		assert.Equal(0, len(logs))
+
+		//filer by wrong address
+		logs, err = ethApi.GetLogs(m.Ctx, filters.FilterCriteria{
+			Topics: [][]common.Hash{{common.HexToHash("0x68f6a0f063c25c6678c443b9a484086f15ba8f91f60218695d32a5251f2050eb")}},
+		})
+		assert.NoError(err)
+		assert.Equal(1, len(logs))
+	}
+	//
+	//api := NewErigonAPI(baseApi, m.DB, nil)
+	//logs, err := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)})
+	//assert.NoError(err)
+	//assert.Equal(uint64(10), logs[0].BlockNumber)
+	//
+	////filer by wrong address
+	//logs, err = api.GetLogs(m.Ctx, filters.FilterCriteria{
+	//	Addresses: common.Addresses{common.Address{}},
+	//})
+	//assert.NoError(err)
+	//assert.Equal(0, len(logs))
+	//
+	////filer by wrong address
+	//logs, err = api.GetLogs(m.Ctx, filters.FilterCriteria{
+	//	Topics: [][]common.Hash{{common.HexToHash("0x68f6a0f063c25c6678c443b9a484086f15ba8f91f60218695d32a5251f2050eb")}},
+	//})
+	//assert.NoError(err)
+	//assert.Equal(1, len(logs))
+}
+
 func TestErigonGetLatestLogs(t *testing.T) {
 	assert := assert.New(t)
 	m, _, _ := rpcdaemontest.CreateTestSentry(t)
@@ -32,7 +81,7 @@ func TestErigonGetLatestLogs(t *testing.T) {
 	db := m.DB
 	agg := m.HistoryV3Components()
 	api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), db, nil)
-	expectedLogs, _ := api.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())})
+	expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())})
 
 	expectedErigonLogs := make([]*types.ErigonLog, 0)
 	for i := len(expectedLogs) - 1; i >= 0; i-- {
@@ -49,8 +98,8 @@ func TestErigonGetLatestLogs(t *testing.T) {
 			Timestamp:   expectedLogs[i].Timestamp,
 		})
 	}
-	actual, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{}, filters.LogFilterOptions{
-		LogCount: uint64((len(expectedLogs))),
+	actual, err := api.GetLatestLogs(m.Ctx, filters.FilterCriteria{}, filters.LogFilterOptions{
+		LogCount: uint64(len(expectedLogs)),
 	})
 	if err != nil {
 		t.Errorf("calling erigon_getLatestLogs: %v", err)
@@ -67,7 +116,7 @@ func TestErigonGetLatestLogsIgnoreTopics(t *testing.T) {
 	db := m.DB
 	agg := m.HistoryV3Components()
 	api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), db, nil)
-	expectedLogs, _ := api.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())})
+	expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())})
 
 	expectedErigonLogs := make([]*types.ErigonLog, 0)
 	for i := len(expectedLogs) - 1; i >= 0; i-- {
@@ -97,7 +146,7 @@ func TestErigonGetLatestLogsIgnoreTopics(t *testing.T) {
 			expectedLogs[i].Topics[0],
 		})
 	}
-	actual, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{Topics: containsTopics}, filters.LogFilterOptions{
+	actual, err := api.GetLatestLogs(m.Ctx, filters.FilterCriteria{Topics: containsTopics}, filters.LogFilterOptions{
 		BlockCount: blockCount,
 	})
 	if err != nil {
@@ -158,33 +207,24 @@ func TestGetBlockReceiptsByBlockHash(t *testing.T) {
 	stateCache := kvcache.New(kvcache.DefaultCoherentConfig)
 	api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil)
 
-	if m.HistoryV3 {
-		t.Skip("ErigonV3 doesn't store receipts in DB. Can't use \"rawdb\" package in this case. need somehow change assertion")
+	expect := map[uint64]string{
+		0: `[]`,
+		1: `[{"blockHash":"0x63b978611c906a61e4a8333fedeea8d62a1c869fc9a19acf6ed0cc5139247eda","blockNumber":"0x1","contractAddress":null,"cumulativeGasUsed":"0x5208","effectiveGasPrice":"0x0","from":"0x71562b71999873db5b286df957af199ec94617f7","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":"0x703c4b2bd70c169f5717101caee543299fc946c7","transactionHash":"0x9ca7a9e6bf23353fc5ac37f5c5676db1accec4af83477ac64cdcaa37f3a837f9","transactionIndex":"0x0","type":"0x0"}]`,
+		2: `[{"blockHash":"0xd3294fcc342ff74be4ae07fb25cd3b2fbb6c2b7830f212ee0723da956e70e099","blockNumber":"0x2","contractAddress":null,"cumulativeGasUsed":"0x5208","effectiveGasPrice":"0x0","from":"0x71562b71999873db5b286df957af199ec94617f7","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":"0x703c4b2bd70c169f5717101caee543299fc946c7","transactionHash":"0xf190eed1578cdcfe69badd05b7ef183397f336dc3de37baa4adbfb4bc657c11e","transactionIndex":"0x0","type":"0x0"},{"blockHash":"0xd3294fcc342ff74be4ae07fb25cd3b2fbb6c2b7830f212ee0723da956e70e099","blockNumber":"0x2","contractAddress":null,"cumulativeGasUsed":"0xa410","effectiveGasPrice":"0x0","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":"0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e","transactionHash":"0x309a030e44058e435a2b01302006880953e2c9319009db97013eb130d7a24eab","transactionIndex":"0x1","type":"0x0"}]`,
+		3: `[]`,
+		4: `[]`,
 	}
 	err := m.DB.View(m.Ctx, func(tx kv.Tx) error {
 		for i := uint64(0); i <= rawdb.ReadCurrentHeader(tx).Number.Uint64(); i++ {
-			block, err := rawdb.ReadBlockByNumber(tx, i)
-			if err != nil {
-				return err
-			}
-
-			r, err := rawdb.ReadReceiptsByHash(tx, block.Hash())
-			if err != nil {
-				return err
-			}
-
-			marshaledReceipts := make([]map[string]interface{}, 0, len(r))
-			for _, receipt := range r {
-				txn := block.Transactions()[receipt.TransactionIndex]
-				marshaledReceipts = append(marshaledReceipts, marshalReceipt(receipt, txn, m.ChainConfig, block, txn.Hash(), true))
-			}
+			block := rawdb.ReadHeaderByNumber(tx, i)
 
 			receiptsFromBlock, err := api.GetBlockReceiptsByBlockHash(context.Background(), block.Hash())
 			if err != nil {
 				return err
 			}
 
-			assert.EqualValues(t, marshaledReceipts, receiptsFromBlock)
+			a, _ := json.Marshal(receiptsFromBlock)
+			assert.Equal(t, expect[block.Number.Uint64()], string(a))
 		}
 		return nil
 	})

cmd/rpcdaemon/commands/eth_receipts.go

@@ -311,7 +311,6 @@ func applyFiltersV3(out *roaring64.Bitmap, tx kv.TemporalTx, begin, end uint64,
 	}
 	toTxNum++
 
-	out.Clear()
 	out.AddRange(fromTxNum, toTxNum) // [from,to)
 	topicsBitmap, err := getTopicsBitmapV3(tx, crit.Topics, fromTxNum, toTxNum)
 	if err != nil {
@@ -484,29 +483,29 @@ func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end
 func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]common.Hash, from, to uint64) (*roaring64.Bitmap, error) {
 	var result *roaring64.Bitmap
 	for _, sub := range topics {
-		var bitmapForORing roaring64.Bitmap
+		bitmapForORing := bitmapdb.NewBitmap64()
+		defer bitmapdb.ReturnToPool64(bitmapForORing)
+
 		for _, topic := range sub {
 			it, err := tx.IndexRange(temporal.LogTopicIdx, topic.Bytes(), from, to)
 			if err != nil {
 				return nil, err
 			}
-			for it.HasNext() {
-				n, err := it.NextBatch()
-				if err != nil {
-					return nil, err
-				}
-				bitmapForORing.AddMany(n)
+			bm, err := it.ToBitmap()
+			if err != nil {
+				return nil, err
 			}
+			bitmapForORing.Or(bm)
 		}
 
 		if bitmapForORing.GetCardinality() == 0 {
 			continue
 		}
 		if result == nil {
-			result = &bitmapForORing
+			result = bitmapForORing.Clone()
 			continue
 		}
-		result = roaring64.And(&bitmapForORing, result)
+		result = roaring64.And(bitmapForORing, result)
 	}
 	return result, nil
 }
@@ -525,15 +524,10 @@ func getAddrsBitmapV3(tx kv.TemporalTx, addrs []common.Address, from, to uint64)
 		if err != nil {
 			return nil, err
 		}
-		m := roaring64.New()
-		for it.HasNext() {
-			n, err := it.NextBatch()
-			if err != nil {
-				return nil, err
-			}
-			m.AddMany(n)
+		rx[idx], err = it.ToBitmap()
+		if err != nil {
+			return nil, err
 		}
-		rx[idx] = m
 	}
 	return roaring64.FastOr(rx...), nil
 }

cmd/rpcdaemon/commands/storage_range.go

@@ -9,12 +9,12 @@ import (
 
 // StorageRangeResult is the result of a debug_storageRangeAt API call.
 type StorageRangeResult struct {
-	Storage StorageMap   `json:"storage"`
+	Storage storageMap   `json:"storage"`
 	NextKey *common.Hash `json:"nextKey"` // nil if Storage includes the last key in the trie.
 }
 
-// StorageMap a map from storage locations to StorageEntry items
-type StorageMap map[common.Hash]StorageEntry
+// storageMap a map from storage locations to StorageEntry items
+type storageMap map[common.Hash]StorageEntry
 
 // StorageEntry an entry in storage of the account
 type StorageEntry struct {
@@ -26,8 +26,8 @@ type walker interface {
 	ForEachStorage(addr common.Address, startLocation common.Hash, cb func(key, seckey common.Hash, value uint256.Int) bool, maxResults int) error
 }
 
-func StorageRangeAt(stateReader walker, contractAddress common.Address, start []byte, maxResult int) (StorageRangeResult, error) {
-	result := StorageRangeResult{Storage: StorageMap{}}
+func storageRangeAt(stateReader walker, contractAddress common.Address, start []byte, maxResult int) (StorageRangeResult, error) {
+	result := StorageRangeResult{Storage: storageMap{}}
 	resultCount := 0
 
 	if err := stateReader.ForEachStorage(contractAddress, common.BytesToHash(start), func(key, seckey common.Hash, value uint256.Int) bool {