Andrew7234/api tests

Makefile

@@ -58,6 +58,18 @@ test-e2e: export OASIS_INDEXER_E2E = true
 test-e2e:
 	@$(GO) test -race -coverpkg=./... -coverprofile=coverage.txt -covermode=atomic -v ./tests/e2e
 
+fill-cache-for-e2e-regression: oasis-indexer
+	cp tests/e2e_regression/e2e_config.yml /tmp/indexer_fill_e2e_regression_cache.yml
+	sed -E -i='' 's/query_on_cache_miss: false/query_on_cache_miss: true/g' /tmp/indexer_fill_e2e_regression_cache.yml
+	./oasis-indexer --config /tmp/indexer_fill_e2e_regression_cache.yml analyze
+
+# Run the api tests locally, assuming the environment is set up with an oasis-node that is 
+# accessible as specified in the config file.
+test-e2e-regression: oasis-indexer
+	./oasis-indexer --config tests/e2e_regression/e2e_config.yml analyze
+	@$(ECHO) "$(CYAN)*** Indexer finished; starting api tests...$(OFF)"
+	./tests/e2e_regression/run.sh
+
 # Format code.
 fmt:
 	@$(ECHO) "$(CYAN)*** Running Go formatters...$(OFF)"

analyzer/consensus/consensus.go

@@ -37,7 +37,7 @@ type EventType = apiTypes.ConsensusEventType // alias for brevity
 
 type parsedEvent struct {
 	ty               EventType
-	body             interface{}
+	rawBodyJSON      json.RawMessage
 	relatedAddresses []staking.Address
 }
 
@@ -482,7 +482,7 @@ func (m *Main) queueTxEventInserts(batch *storage.QueryBatch, data *storage.Cons
 			eventData := m.extractEventData(event)
 			txAccounts = append(txAccounts, eventData.relatedAddresses...)
 			accounts := extractUniqueAddresses(eventData.relatedAddresses)
-			body, err := json.Marshal(eventData.body)
+			body, err := json.Marshal(eventData.rawBodyJSON)
 			if err != nil {
 				return err
 			}
@@ -915,7 +915,7 @@ func (m *Main) queueGovernanceEventInserts(batch *storage.QueryBatch, data *stor
 
 func (m *Main) queueSingleEventInserts(batch *storage.QueryBatch, eventData *parsedEvent, height int64) error {
 	accounts := extractUniqueAddresses(eventData.relatedAddresses)
-	body, err := json.Marshal(eventData.body)
+	body, err := json.Marshal(eventData.rawBodyJSON)
 	if err != nil {
 		return err
 	}
@@ -950,8 +950,8 @@ func extractUniqueAddresses(accounts []staking.Address) []string {
 // extractEventData extracts the type, the body (JSON-serialized), and the related accounts of an event.
 func (m *Main) extractEventData(event nodeapi.Event) parsedEvent {
 	eventData := parsedEvent{
-		ty:   event.Type,
-		body: event.Body,
+		ty:          event.Type,
+		rawBodyJSON: event.RawBodyJSON,
 	}
 
 	// Fill in related accounts.

analyzer/runtime/extract.go

@@ -248,7 +248,7 @@ func registerTokenDecrease(tokenChanges map[TokenChangeKey]*big.Int, contractAdd
 	change.Sub(change, amount)
 }
 
-func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.RuntimeTransactionWithResults, rawEvents []*nodeapi.RuntimeEvent, logger *log.Logger) (*BlockData, error) { //nolint:gocyclo
+func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []nodeapi.RuntimeTransactionWithResults, rawEvents []nodeapi.RuntimeEvent, logger *log.Logger) (*BlockData, error) { //nolint:gocyclo
 	blockData := BlockData{
 		Header:              blockHeader,
 		NumTransactions:     len(txrs),
@@ -260,7 +260,7 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.Runtim
 	}
 
 	// Extract info from non-tx events.
-	rawNonTxEvents := []*nodeapi.RuntimeEvent{}
+	rawNonTxEvents := []nodeapi.RuntimeEvent{}
 	for _, e := range rawEvents {
 		if e.TxHash.String() == util.ZeroTxHash {
 			rawNonTxEvents = append(rawNonTxEvents, e)
@@ -394,9 +394,9 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.Runtim
 			}
 			blockTransactionData.Amount = common.Ptr(common.BigIntFromQuantity(amount))
 		}
-		txEvents := make([]*nodeapi.RuntimeEvent, len(txr.Events))
+		txEvents := make([]nodeapi.RuntimeEvent, len(txr.Events))
 		for i, e := range txr.Events {
-			txEvents[i] = (*nodeapi.RuntimeEvent)(e)
+			txEvents[i] = (nodeapi.RuntimeEvent)(*e)
 		}
 		extractedTxEvents, err := extractEvents(&blockData, blockTransactionData.RelatedAccountAddresses, txEvents)
 		if err != nil {
@@ -452,7 +452,7 @@ func sumGasUsed(events []*EventData) (sum uint64, foundGasUsedEvent bool) {
 	return
 }
 
-func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Address]bool, eventsRaw []*nodeapi.RuntimeEvent) ([]*EventData, error) {
+func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Address]bool, eventsRaw []nodeapi.RuntimeEvent) ([]*EventData, error) {
 	extractedEvents := []*EventData{}
 	if err := VisitSdkEvents(eventsRaw, &SdkEventHandler{
 		Core: func(event *core.Event) error {

analyzer/runtime/visitors.go

@@ -151,9 +151,9 @@ func VisitSdkEvent(event *nodeapi.RuntimeEvent, handler *SdkEventHandler) error
 	return nil
 }
 
-func VisitSdkEvents(events []*nodeapi.RuntimeEvent, handler *SdkEventHandler) error {
-	for i, event := range events {
-		if err := VisitSdkEvent(event, handler); err != nil {
+func VisitSdkEvents(events []nodeapi.RuntimeEvent, handler *SdkEventHandler) error {
+	for i := range events {
+		if err := VisitSdkEvent(&events[i], handler); err != nil {
 			return fmt.Errorf("event %d: %w", i, err)
 		}
 	}

common/types.go

@@ -1,6 +1,7 @@
 package common
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
@@ -90,6 +91,16 @@ func Ptr[T any](v T) *T {
 	return &v
 }
 
+// Returns `v` as a JSON string. If `v` cannot be marshaled,
+// returns the string "null" instead.
+func TryAsJSON(v interface{}) json.RawMessage {
+	encoded, err := json.Marshal(v)
+	if err != nil {
+		return json.RawMessage("null")
+	}
+	return json.RawMessage(encoded)
+}
+
 // Key used to set values in a web request context. API uses this to set
 // values, backend uses this to retrieve values.
 type ContextKey string

config/config.go

@@ -117,6 +117,9 @@ type AnalyzersList struct {
 
 // SourceConfig has some controls about what chain we're analyzing and how to connect.
 type SourceConfig struct {
+	// Cache holds the configuration for a file-based caching backend.
+	Cache *CacheConfig `koanf:"cache"`
+
 	// ChainName is the name of the chain (e.g. mainnet/testnet). Set
 	// this to use one of the default chains.
 	ChainName string `koanf:"chain_name"`
@@ -168,6 +171,22 @@ func SingleNetworkLookup(rpc string) map[string]*NodeConfig {
 	}
 }
 
+type CacheConfig struct {
+	// CacheDir is the directory where the cache data is stored
+	CacheDir string `koanf:"cache_dir"`
+
+	// If set, the indexer will query the node upon any cache
+	// misses.
+	QueryOnCacheMiss bool `koanf:"query_on_cache_miss"`
+}
+
+func (cfg *CacheConfig) Validate() error {
+	if cfg.CacheDir == "" {
+		return fmt.Errorf("invalid cache filepath")
+	}
+	return nil
+}
+
 type CustomChainConfig struct {
 	// History is the sequence of networks in the chain.
 	History *History `koanf:"history"`

go.mod

@@ -134,6 +134,7 @@ require (
 )
 
 require (
+	github.com/akrylysov/pogreb v0.10.1
 	github.com/deepmap/oapi-codegen v1.12.4
 	github.com/oasisprotocol/metadata-registry-tools v0.0.0-20220406100644-7e9a2b991920
 	github.com/rs/cors v1.8.3

go.sum

@@ -117,6 +117,8 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIO
 github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
+github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=

storage/api.go

@@ -233,8 +233,8 @@ type RuntimeSourceStorage interface {
 type RuntimeAllData struct {
 	Round                   uint64
 	BlockHeader             nodeapi.RuntimeBlockHeader
-	RawEvents               []*nodeapi.RuntimeEvent
-	TransactionsWithResults []*nodeapi.RuntimeTransactionWithResults
+	RawEvents               []nodeapi.RuntimeEvent
+	TransactionsWithResults []nodeapi.RuntimeTransactionWithResults
 }
 
 // TransactionWithResults contains a verified transaction, and the results of

storage/oasis/client.go

@@ -5,9 +5,12 @@ package oasis
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 
 	"github.com/oasisprotocol/oasis-indexer/common"
 	"github.com/oasisprotocol/oasis-indexer/config"
+	"github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi"
+	"github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi/file"
 	"github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi/history"
 )
 
@@ -17,10 +20,32 @@ const (
 
 // NewConsensusClient creates a new ConsensusClient.
 func NewConsensusClient(ctx context.Context, sourceConfig *config.SourceConfig) (*ConsensusClient, error) {
+	// If we are using purely file-backed indexer, do not connect to the node.
+	if sourceConfig.Cache != nil && !sourceConfig.Cache.QueryOnCacheMiss {
+		cachePath := filepath.Join(sourceConfig.Cache.CacheDir, "consensus")
+		nodeApi, err := file.NewFileConsensusApiLite(cachePath, nil)
+		if err != nil {
+			return nil, fmt.Errorf("error instantiating cache-based consensusApi: %w", err)
+		}
+		return &ConsensusClient{
+			nodeApi: nodeApi,
+			network: sourceConfig.SDKNetwork(),
+		}, nil
+	}
+
+	// Create an API that connects to the real node, then wrap it in a caching layer.
+	var nodeApi nodeapi.ConsensusApiLite
 	nodeApi, err := history.NewHistoryConsensusApiLite(ctx, sourceConfig.History(), sourceConfig.Nodes, sourceConfig.FastStartup)
 	if err != nil {
 		return nil, fmt.Errorf("instantiating history consensus API lite: %w", err)
 	}
+	if sourceConfig.Cache != nil {
+		cachePath := filepath.Join(sourceConfig.Cache.CacheDir, "consensus")
+		nodeApi, err = file.NewFileConsensusApiLite(cachePath, nodeApi)
+		if err != nil {
+			return nil, fmt.Errorf("error instantiating cache-based consensusApi: %w", err)
+		}
+	}
 	return &ConsensusClient{
 		nodeApi: nodeApi,
 		network: sourceConfig.SDKNetwork(),
@@ -30,11 +55,26 @@ func NewConsensusClient(ctx context.Context, sourceConfig *config.SourceConfig)
 // NewRuntimeClient creates a new RuntimeClient.
 func NewRuntimeClient(ctx context.Context, sourceConfig *config.SourceConfig, runtime common.Runtime) (*RuntimeClient, error) {
 	sdkPT := sourceConfig.SDKNetwork().ParaTimes.All[runtime.String()]
+	var nodeApi nodeapi.RuntimeApiLite
 	nodeApi, err := history.NewHistoryRuntimeApiLite(ctx, sourceConfig.History(), sdkPT, sourceConfig.Nodes, sourceConfig.FastStartup, runtime)
 	if err != nil {
 		return nil, fmt.Errorf("instantiating history runtime API lite: %w", err)
 	}
 
+	// todo: short circuit if using purely a file-based backend and avoid connecting
+	// to the node at all. this requires storing runtime info offline.
+	if sourceConfig.Cache != nil {
+		cachePath := filepath.Join(sourceConfig.Cache.CacheDir, runtime.String())
+		if sourceConfig.Cache.QueryOnCacheMiss {
+			nodeApi, err = file.NewFileRuntimeApiLite(runtime.String(), cachePath, nodeApi)
+		} else {
+			nodeApi, err = file.NewFileRuntimeApiLite(runtime.String(), cachePath, nil)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("error instantiating cache-based runtimeApi: %w", err)
+		}
+	}
+
 	return &RuntimeClient{
 		nodeApi: nodeApi,
 		sdkPT:   sdkPT,

storage/oasis/nodeapi/api.go

@@ -2,6 +2,7 @@ package nodeapi
 
 import (
 	"context"
+	"encoding/json"
 	"time"
 
 	beacon "github.com/oasisprotocol/oasis-core/go/beacon/api"
@@ -82,9 +83,14 @@ type Event struct {
 	Height int64
 	TxHash hash.Hash
 
-	// The fine-grained Event struct directly from oasis-core; corresponds to
-	// at most one of the fields below (StakingTransfer, StakingBurn, etc.).
-	Body interface{}
+	// The body of the Event struct as it was received from oasis-core. For most
+	// event types, a summary of the event (containing only indexer-relevant fields)
+	// will be present in one of the fields below (StakingTransfer, StakingBurn, etc.).
+	// For event types that the indexer doesn't process beyond logging, only this
+	// field will be populated.
+	// We convert to JSON and effectively erase the type here in order to decouple
+	// oasis-core types (which vary between versions) from the indexer.
+	RawBodyJSON json.RawMessage
 
 	// Called "Kind" in oasis-core but "Type" in indexer APIs and DBs.
 	Type apiTypes.ConsensusEventType
@@ -192,10 +198,10 @@ type Proposal governance.Proposal
 
 // Like ConsensusApiLite, but for the runtime API.
 type RuntimeApiLite interface {
-	GetEventsRaw(ctx context.Context, round uint64) ([]*RuntimeEvent, error)
+	GetEventsRaw(ctx context.Context, round uint64) ([]RuntimeEvent, error)
 	EVMSimulateCall(ctx context.Context, round uint64, gasPrice []byte, gasLimit uint64, caller []byte, address []byte, value []byte, data []byte) ([]byte, error)
 	GetBlockHeader(ctx context.Context, round uint64) (*RuntimeBlockHeader, error)
-	GetTransactionsWithResults(ctx context.Context, round uint64) ([]*RuntimeTransactionWithResults, error)
+	GetTransactionsWithResults(ctx context.Context, round uint64) ([]RuntimeTransactionWithResults, error)
 }
 
 type (