feat: add erigon_getLogs with timestamp field to erigon rpcdaemon a…

…nd fix the issue 4982 (#4968)

* add_abigen_error_handle

* add abigen error type test code

* add field timestamp in `eth_getLogs` api

add field timestamp in `eth_getLogs` api

* undo add field timestamp in `eth_getLogs`

* add `erigon_getLogs` api and add field `timestamp`

add `erigon_getLogs` api and add field `timestamp`

* feat: add `erigon_getLogs` with timestamp field to erigon rpcdaemon

feat: add `erigon_getLogs` with timestamp field to erigon rpcdaemon

* fix: issue `4982` roaring out of range

fix: issue 4982 roaring out of range

* convert rangeEnd to latest

convert rangeEnd to latest when range end is a big value that go out of range of MaxUint32

* add begin condition

add begin condition in case of bigger than latest block

* add annotation to unreachable code

cmd/rpcdaemon/cli/config.go

@@ -284,7 +284,8 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger,
 		borDb = borKv
 	} else {
 		if cfg.StateCache.KeysLimit > 0 {
-			stateCache = kvcache.New(cfg.StateCache)
+			panic(1)
+			//stateCache = kvcache.New(cfg.StateCache)
 		} else {
 			stateCache = kvcache.NewDummy()
 		}

cmd/rpcdaemon/commands/erigon_api.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"context"
+	ethFilters "github.com/ledgerwatch/erigon/eth/filters"
 
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon/common"
@@ -26,6 +27,7 @@ type ErigonAPI interface {
 	// Receipt related (see ./erigon_receipts.go)
 	GetLogsByHash(ctx context.Context, hash common.Hash) ([][]*types.Log, error)
 	//GetLogsByNumber(ctx context.Context, number rpc.BlockNumber) ([][]*types.Log, error)
+	GetLogs(ctx context.Context, crit ethFilters.FilterCriteria) ([]*types.Log, error)
 
 	// WatchTheBurn / reward related (see ./erigon_issuance.go)
 	WatchTheBurn(ctx context.Context, blockNr rpc.BlockNumber) (Issuance, error)

cmd/rpcdaemon/commands/erigon_receipts.go

@@ -1,11 +1,21 @@
 package commands
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
 	"fmt"
-
+	"github.com/RoaringBitmap/roaring"
+	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon/common"
+	"github.com/ledgerwatch/erigon/common/dbutils"
+	"github.com/ledgerwatch/erigon/core/rawdb"
 	"github.com/ledgerwatch/erigon/core/types"
+	"github.com/ledgerwatch/erigon/eth/filters"
+	"github.com/ledgerwatch/erigon/ethdb/bitmapdb"
+	"github.com/ledgerwatch/erigon/ethdb/cbor"
+	"github.com/ledgerwatch/erigon/rpc"
+	"github.com/ledgerwatch/erigon/turbo/rpchelper"
 )
 
 // GetLogsByHash implements erigon_getLogsByHash. Returns an array of arrays of logs generated by the transactions in the block given by the block's hash.
@@ -40,6 +50,165 @@ func (api *ErigonImpl) GetLogsByHash(ctx context.Context, hash common.Hash) ([][
 	return logs, nil
 }
 
+// GetLogs implements eth_getLogs. Returns an array of logs matching a given filter object.
+func (api *ErigonImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) ([]*types.Log, error) {
+	var begin, end uint64
+	logs := []*types.Log{}
+
+	tx, beginErr := api.db.BeginRo(ctx)
+	if beginErr != nil {
+		return logs, beginErr
+	}
+	defer tx.Rollback()
+
+	if crit.BlockHash != nil {
+		number := rawdb.ReadHeaderNumber(tx, *crit.BlockHash)
+		if number == nil {
+			return nil, fmt.Errorf("block not found: %x", *crit.BlockHash)
+		}
+		begin = *number
+		end = *number
+	} else {
+		// Convert the RPC block numbers into internal representations
+		latest, err := rpchelper.GetLatestBlockNumber(tx)
+		if err != nil {
+			return nil, err
+		}
+
+		begin = latest
+		if crit.FromBlock != nil {
+			if crit.FromBlock.Sign() >= 0 {
+				begin = crit.FromBlock.Uint64()
+			} else if !crit.FromBlock.IsInt64() || crit.FromBlock.Int64() != int64(rpc.LatestBlockNumber) {
+				return nil, fmt.Errorf("negative value for FromBlock: %v", crit.FromBlock)
+			}
+		}
+		end = latest
+		if crit.ToBlock != nil {
+			if crit.ToBlock.Sign() >= 0 {
+				end = crit.ToBlock.Uint64()
+			} else if !crit.ToBlock.IsInt64() || crit.ToBlock.Int64() != int64(rpc.LatestBlockNumber) {
+				return nil, fmt.Errorf("negative value for ToBlock: %v", crit.ToBlock)
+			}
+		}
+	}
+	if end < begin {
+		return nil, fmt.Errorf("end (%d) < begin (%d)", end, begin)
+	}
+	if end > roaring.MaxUint32 {
+		return nil, fmt.Errorf("end (%d) > MaxUint32", end)
+	}
+	blockNumbers := roaring.New()
+	blockNumbers.AddRange(begin, end+1) // [min,max)
+
+	topicsBitmap, err := getTopicsBitmap(tx, crit.Topics, uint32(begin), uint32(end))
+	if err != nil {
+		return nil, err
+	}
+	if topicsBitmap != nil {
+		blockNumbers.And(topicsBitmap)
+	}
+
+	var addrBitmap *roaring.Bitmap
+	for _, addr := range crit.Addresses {
+		m, err := bitmapdb.Get(tx, kv.LogAddressIndex, addr[:], uint32(begin), uint32(end))
+		if err != nil {
+			return nil, err
+		}
+		if addrBitmap == nil {
+			addrBitmap = m
+			continue
+		}
+		addrBitmap = roaring.Or(addrBitmap, m)
+	}
+
+	if addrBitmap != nil {
+		blockNumbers.And(addrBitmap)
+	}
+
+	if blockNumbers.GetCardinality() == 0 {
+		return logs, nil
+	}
+
+	iter := blockNumbers.Iterator()
+	for iter.HasNext() {
+		if err = ctx.Err(); err != nil {
+			return nil, err
+		}
+
+		blockNumber := uint64(iter.Next())
+		var logIndex uint
+		var txIndex uint
+		var blockLogs []*types.Log
+		err := tx.ForPrefix(kv.Log, dbutils.EncodeBlockNumber(blockNumber), func(k, v []byte) error {
+			var logs types.Logs
+			if err := cbor.Unmarshal(&logs, bytes.NewReader(v)); err != nil {
+				return fmt.Errorf("receipt unmarshal failed:  %w", err)
+			}
+			for _, log := range logs {
+				log.Index = logIndex
+				logIndex++
+			}
+			filtered := filterLogs(logs, crit.Addresses, crit.Topics)
+			if len(filtered) == 0 {
+				return nil
+			}
+			txIndex = uint(binary.BigEndian.Uint32(k[8:]))
+			for _, log := range filtered {
+				log.TxIndex = txIndex
+			}
+			blockLogs = append(blockLogs, filtered...)
+
+			return nil
+		})
+		if err != nil {
+			return logs, err
+		}
+		if len(blockLogs) == 0 {
+			continue
+		}
+
+		header, err := api._blockReader.HeaderByNumber(ctx, tx, blockNumber)
+		if err != nil {
+			return nil, err
+		}
+		if header == nil {
+			return nil, fmt.Errorf("block header not found: %d", blockNumber)
+		}
+		timestamp := header.Time
+
+		blockHash, err := rawdb.ReadCanonicalHash(tx, blockNumber)
+		if err != nil {
+			return nil, err
+		}
+
+		body, err := api._blockReader.BodyWithTransactions(ctx, tx, blockHash, blockNumber)
+		if err != nil {
+			return nil, err
+		}
+		if body == nil {
+			return nil, fmt.Errorf("block not found %d", blockNumber)
+		}
+		for _, log := range blockLogs {
+			log.Timestamp = timestamp
+			log.BlockNumber = blockNumber
+			log.BlockHash = blockHash
+			log.TxHash = body.Transactions[log.TxIndex].Hash()
+		}
+		logs = append(logs, blockLogs...)
+
+		borLogs := rawdb.ReadBorReceiptLogs(tx, blockHash, blockNumber, txIndex+1, logIndex)
+		if borLogs != nil {
+			borLogs = filterLogs(borLogs, crit.Addresses, crit.Topics)
+			if len(borLogs) > 0 {
+				logs = append(logs, borLogs...)
+			}
+		}
+	}
+
+	return logs, nil
+}
+
 // GetLogsByNumber implements erigon_getLogsByHash. Returns all the logs that appear in a block given the block's hash.
 // func (api *ErigonImpl) GetLogsByNumber(ctx context.Context, number rpc.BlockNumber) ([][]*types.Log, error) {
 // 	tx, err := api.db.Begin(ctx, false)

cmd/rpcdaemon/commands/eth_receipts.go

@@ -119,7 +119,16 @@ func (api *APIImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) ([
 	if end < begin {
 		return nil, fmt.Errorf("end (%d) < begin (%d)", end, begin)
 	}
-
+	if end > roaring.MaxUint32 {
+		latest, err := rpchelper.GetLatestBlockNumber(tx)
+		if err != nil {
+			return nil, err
+		}
+		if begin > latest {
+			return nil, fmt.Errorf("begin (%d) > latest (%d)", begin, latest)
+		}
+		end = latest
+	}
 	blockNumbers := roaring.New()
 	blockNumbers.AddRange(begin, end+1) // [min,max)
 	topicsBitmap, err := getTopicsBitmap(tx, crit.Topics, uint32(begin), uint32(end))