diff --git a/analyzer/queries/queries.go b/analyzer/queries/queries.go index 0d9b68752..b5964a5f4 100644 --- a/analyzer/queries/queries.go +++ b/analyzer/queries/queries.go @@ -256,8 +256,8 @@ const ( VALUES ($1, $2, $3, $4)` RuntimeTransactionInsert = ` - INSERT INTO chain.runtime_transactions (runtime, round, tx_index, tx_hash, tx_eth_hash, gas_used, size, timestamp, raw, result_raw) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)` + INSERT INTO chain.runtime_transactions (runtime, round, tx_index, tx_hash, tx_eth_hash, fee, gas_limit, gas_used, size, timestamp, method, body, "to", amount, success, error_module, error_code, error_message) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)` RuntimeEventInsert = ` INSERT INTO chain.runtime_events (runtime, round, tx_index, tx_hash, type, body, evm_log_name, evm_log_params, related_accounts) diff --git a/analyzer/runtime/extract.go b/analyzer/runtime/extract.go index 90b58ff09..52fa30b8f 100644 --- a/analyzer/runtime/extract.go +++ b/analyzer/runtime/extract.go @@ -15,15 +15,17 @@ import ( ethCommon "github.com/ethereum/go-ethereum/common" "github.com/oasisprotocol/oasis-core/go/common/cbor" "github.com/oasisprotocol/oasis-core/go/common/crypto/address" + "github.com/oasisprotocol/oasis-core/go/common/quantity" "github.com/oasisprotocol/oasis-sdk/client-sdk/go/modules/accounts" "github.com/oasisprotocol/oasis-sdk/client-sdk/go/modules/consensusaccounts" "github.com/oasisprotocol/oasis-sdk/client-sdk/go/modules/core" "github.com/oasisprotocol/oasis-sdk/client-sdk/go/modules/evm" sdkTypes "github.com/oasisprotocol/oasis-sdk/client-sdk/go/types" - common "github.com/oasisprotocol/oasis-indexer/analyzer/uncategorized" + uncategorized "github.com/oasisprotocol/oasis-indexer/analyzer/uncategorized" "github.com/oasisprotocol/oasis-indexer/analyzer/util" apiTypes "github.com/oasisprotocol/oasis-indexer/api/v1/types" + "github.com/oasisprotocol/oasis-indexer/common" "github.com/oasisprotocol/oasis-indexer/log" "github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi" ) @@ -44,6 +46,22 @@ type BlockTransactionData struct { RawResult []byte SignerData []*BlockTransactionSignerData RelatedAccountAddresses map[apiTypes.Address]bool + Fee common.BigInt + GasLimit uint64 + Method string + Body interface{} + To *apiTypes.Address // Extracted from the body for convenience. Semantics vary by tx type. + Amount *common.BigInt // Extracted from the body for convenience. Semantics vary by tx type. + Success *bool + Error *TxError +} + +type TxError struct { + Code uint32 + Module string + // `Module` should be present but `Message` may be null + // https://github.com/oasisprotocol/oasis-sdk/blob/fb741678585c04fdb413441f2bfba18aafbf98f3/client-sdk/go/types/transaction.go#L488-L492 + Message *string } type EventBody interface{} @@ -121,7 +139,7 @@ func extractAddressPreimage(as *sdkTypes.AddressSpec) (*AddressPreimageData, err // Use a scheme such that we can compute Secp256k1 addresses from Ethereum // addresses as this makes things more interoperable. untaggedPk, _ := spec.Secp256k1Eth.MarshalBinaryUncompressedUntagged() - data = common.SliceEthAddress(common.Keccak256(untaggedPk)) + data = uncategorized.SliceEthAddress(uncategorized.Keccak256(untaggedPk)) case spec.Sr25519 != nil: ctx = sdkTypes.AddressV0Sr25519Context data, _ = spec.Sr25519.MarshalBinary() @@ -143,7 +161,7 @@ func extractAddressPreimage(as *sdkTypes.AddressSpec) (*AddressPreimageData, err } func registerAddressSpec(addressPreimages map[apiTypes.Address]*AddressPreimageData, as *sdkTypes.AddressSpec) (apiTypes.Address, error) { - addr, err := common.StringifyAddressSpec(as) + addr, err := uncategorized.StringifyAddressSpec(as) if err != nil { return "", err } @@ -160,7 +178,7 @@ func registerAddressSpec(addressPreimages map[apiTypes.Address]*AddressPreimageD } func registerEthAddress(addressPreimages map[apiTypes.Address]*AddressPreimageData, ethAddr []byte) (apiTypes.Address, error) { - addr, err := common.StringifyEthAddress(ethAddr) + addr, err := uncategorized.StringifyEthAddress(ethAddr) if err != nil { return "", err } @@ -177,7 +195,7 @@ func registerEthAddress(addressPreimages map[apiTypes.Address]*AddressPreimageDa } func registerRelatedSdkAddress(relatedAddresses map[apiTypes.Address]bool, sdkAddr *sdkTypes.Address) (apiTypes.Address, error) { - addr, err := common.StringifySdkAddress(sdkAddr) + addr, err := uncategorized.StringifySdkAddress(sdkAddr) if err != nil { return "", err } @@ -229,7 +247,7 @@ func registerTokenDecrease(tokenChanges map[TokenChangeKey]*big.Int, contractAdd change.Sub(change, amount) } -func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.RuntimeTransactionWithResults, rawEvents []*nodeapi.RuntimeEvent, logger *log.Logger) (*BlockData, error) { +func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.RuntimeTransactionWithResults, rawEvents []*nodeapi.RuntimeEvent, logger *log.Logger) (*BlockData, error) { //nolint:gocyclo blockData := BlockData{ Header: blockHeader, NumTransactions: len(txrs), @@ -259,7 +277,7 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.Runtim blockTransactionData.Index = txIndex blockTransactionData.Hash = txr.Tx.Hash().Hex() if len(txr.Tx.AuthProofs) == 1 && txr.Tx.AuthProofs[0].Module == "evm.ethereum.v0" { - ethHash := hex.EncodeToString(common.Keccak256(txr.Tx.Body)) + ethHash := hex.EncodeToString(uncategorized.Keccak256(txr.Tx.Body)) blockTransactionData.EthHash = ðHash } blockTransactionData.Raw = cbor.Marshal(txr.Tx) @@ -267,7 +285,7 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.Runtim blockTransactionData.Size = len(blockTransactionData.Raw) blockTransactionData.RawResult = cbor.Marshal(txr.Result) blockTransactionData.RelatedAccountAddresses = map[apiTypes.Address]bool{} - tx, err := common.OpenUtxNoVerify(&txr.Tx) + tx, err := uncategorized.OpenUtxNoVerify(&txr.Tx) if err != nil { logger.Error("error decoding tx, skipping tx-specific analysis", "round", blockHeader.Round, @@ -277,7 +295,7 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.Runtim ) tx = nil } - if tx != nil { + if tx != nil { //nolint:nestif blockTransactionData.SignerData = make([]*BlockTransactionSignerData, 0, len(tx.AuthInfo.SignerInfo)) for j, si := range tx.AuthInfo.SignerInfo { var blockTransactionSignerData BlockTransactionSignerData @@ -290,36 +308,75 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.Runtim blockTransactionSignerData.Nonce = int(si.Nonce) blockTransactionData.SignerData = append(blockTransactionData.SignerData, &blockTransactionSignerData) } + blockTransactionData.Fee = common.BigIntFromQuantity(tx.AuthInfo.Fee.Amount.Amount) + blockTransactionData.GasLimit = tx.AuthInfo.Fee.Gas + + // Parse the success/error status. + if fail := txr.Result.Failed; fail != nil { //nolint:gocritic + blockTransactionData.Success = common.Ptr(false) + blockTransactionData.Error = &TxError{ + Code: fail.Code, + Module: fail.Module, + } + if len(fail.Message) > 0 { + blockTransactionData.Error.Message = &fail.Message + } + } else if txr.Result.Ok != nil { + blockTransactionData.Success = common.Ptr(true) + } else { + blockTransactionData.Success = nil + } + + blockTransactionData.Method = tx.Call.Method + var to apiTypes.Address + var amount quantity.Quantity if err = VisitCall(&tx.Call, &txr.Result, &CallHandler{ AccountsTransfer: func(body *accounts.Transfer) error { - if _, err = registerRelatedSdkAddress(blockTransactionData.RelatedAccountAddresses, &body.To); err != nil { + blockTransactionData.Body = body + amount = body.Amount.Amount + if to, err = registerRelatedSdkAddress(blockTransactionData.RelatedAccountAddresses, &body.To); err != nil { return fmt.Errorf("to: %w", err) } return nil }, ConsensusAccountsDeposit: func(body *consensusaccounts.Deposit) error { + blockTransactionData.Body = body + amount = body.Amount.Amount if body.To != nil { - if _, err = registerRelatedSdkAddress(blockTransactionData.RelatedAccountAddresses, body.To); err != nil { + if to, err = registerRelatedSdkAddress(blockTransactionData.RelatedAccountAddresses, body.To); err != nil { return fmt.Errorf("to: %w", err) } } return nil }, ConsensusAccountsWithdraw: func(body *consensusaccounts.Withdraw) error { - // .To is from another chain, so exclude? + blockTransactionData.Body = body + amount = body.Amount.Amount + if body.To != nil { + // Beware, this is the address of an account in the consensus + // layer, not an account in the runtime that generated this event. + // We do not register it as a preimage. + if to, err = uncategorized.StringifySdkAddress(body.To); err != nil { + return fmt.Errorf("to: %w", err) + } + } return nil }, EVMCreate: func(body *evm.Create, ok *[]byte) error { + blockTransactionData.Body = body + amount = uncategorized.QuantityFromBytes(body.Value) if !txr.Result.IsUnknown() && txr.Result.IsSuccess() && len(*ok) == 32 { // todo: is this rigorous enough? - if _, err = registerRelatedEthAddress(blockData.AddressPreimages, blockTransactionData.RelatedAccountAddresses, common.SliceEthAddress(*ok)); err != nil { + if to, err = registerRelatedEthAddress(blockData.AddressPreimages, blockTransactionData.RelatedAccountAddresses, uncategorized.SliceEthAddress(*ok)); err != nil { return fmt.Errorf("created contract: %w", err) } } return nil }, EVMCall: func(body *evm.Call, ok *[]byte) error { - if _, err = registerRelatedEthAddress(blockData.AddressPreimages, blockTransactionData.RelatedAccountAddresses, body.Address); err != nil { + blockTransactionData.Body = body + amount = uncategorized.QuantityFromBytes(body.Value) + if to, err = registerRelatedEthAddress(blockData.AddressPreimages, blockTransactionData.RelatedAccountAddresses, body.Address); err != nil { return fmt.Errorf("address: %w", err) } // todo: maybe parse known token methods @@ -328,6 +385,10 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []*nodeapi.Runtim }); err != nil { return nil, fmt.Errorf("tx %d: %w", txIndex, err) } + if to != "" { + blockTransactionData.To = &to + } + blockTransactionData.Amount = common.Ptr(common.BigIntFromQuantity(amount)) } txEvents := make([]*nodeapi.RuntimeEvent, len(txr.Events)) for i, e := range txr.Events { @@ -492,8 +553,8 @@ func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Ad ERC20Transfer: func(fromEthAddr []byte, toEthAddr []byte, amountU256 []byte) error { amount := &big.Int{} amount.SetBytes(amountU256) - fromZero := bytes.Equal(fromEthAddr, common.ZeroEthAddr) - toZero := bytes.Equal(toEthAddr, common.ZeroEthAddr) + fromZero := bytes.Equal(fromEthAddr, uncategorized.ZeroEthAddr) + toZero := bytes.Equal(toEthAddr, uncategorized.ZeroEthAddr) if !fromZero { fromAddr, err2 := registerRelatedEthAddress(blockData.AddressPreimages, relatedAccountAddresses, fromEthAddr) if err2 != nil { @@ -551,14 +612,14 @@ func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Ad return nil }, ERC20Approval: func(ownerEthAddr []byte, spenderEthAddr []byte, amountU256 []byte) error { - if !bytes.Equal(ownerEthAddr, common.ZeroEthAddr) { + if !bytes.Equal(ownerEthAddr, uncategorized.ZeroEthAddr) { ownerAddr, err2 := registerRelatedEthAddress(blockData.AddressPreimages, relatedAccountAddresses, ownerEthAddr) if err2 != nil { return fmt.Errorf("owner: %w", err2) } eventRelatedAddresses[ownerAddr] = true } - if !bytes.Equal(spenderEthAddr, common.ZeroEthAddr) { + if !bytes.Equal(spenderEthAddr, uncategorized.ZeroEthAddr) { spenderAddr, err2 := registerRelatedEthAddress(blockData.AddressPreimages, relatedAccountAddresses, spenderEthAddr) if err2 != nil { return fmt.Errorf("spender: %w", err2) diff --git a/analyzer/runtime/runtime.go b/analyzer/runtime/runtime.go index db7a61acb..9301fd724 100644 --- a/analyzer/runtime/runtime.go +++ b/analyzer/runtime/runtime.go @@ -289,6 +289,14 @@ func (m *Main) queueDbUpdates(batch *storage.QueryBatch, data *BlockData) { for addr := range transactionData.RelatedAccountAddresses { batch.Queue(queries.RuntimeRelatedTransactionInsert, m.runtime, addr, data.Header.Round, transactionData.Index) } + var error_module string + var error_code uint32 + var error_message *string + if transactionData.Error != nil { + error_module = transactionData.Error.Module + error_code = transactionData.Error.Code + error_message = transactionData.Error.Message + } batch.Queue( queries.RuntimeTransactionInsert, m.runtime, @@ -296,11 +304,19 @@ func (m *Main) queueDbUpdates(batch *storage.QueryBatch, data *BlockData) { transactionData.Index, transactionData.Hash, transactionData.EthHash, + &transactionData.Fee, // pgx bug? Needs a *BigInt (not BigInt) to know how to serialize. + transactionData.GasLimit, transactionData.GasUsed, transactionData.Size, data.Header.Timestamp, - transactionData.Raw, - transactionData.RawResult, + transactionData.Method, + transactionData.Body, + transactionData.To, + transactionData.Amount, + transactionData.Success, + error_module, + error_code, + error_message, ) } diff --git a/analyzer/uncategorized/quantities.go b/analyzer/uncategorized/quantities.go index 9f54337ab..70eb1546b 100644 --- a/analyzer/uncategorized/quantities.go +++ b/analyzer/uncategorized/quantities.go @@ -4,6 +4,7 @@ import ( "fmt" "math/big" + "github.com/oasisprotocol/oasis-core/go/common/quantity" sdkTypes "github.com/oasisprotocol/oasis-sdk/client-sdk/go/types" ) @@ -17,3 +18,9 @@ func StringifyNativeDenomination(amount *sdkTypes.BaseUnits) (string, error) { func StringifyBytes(value []byte) string { return new(big.Int).SetBytes(value).String() } + +func QuantityFromBytes(value []byte) quantity.Quantity { + q := *quantity.NewQuantity() + _ = q.FromBigInt(new(big.Int).SetBytes(value)) + return q +} diff --git a/api/spec/v1.yaml b/api/spec/v1.yaml index 3f27a0d70..75786c2cd 100644 --- a/api/spec/v1.yaml +++ b/api/spec/v1.yaml @@ -1250,13 +1250,13 @@ components: type: object required: [code] properties: + module: + type: string + description: The module of a failed transaction. code: type: integer format: uint32 description: The status code of a failed transaction. - module: - type: string - description: The module of a failed transaction. message: type: string description: The message of a failed transaction. @@ -1983,7 +1983,7 @@ components: RuntimeTransaction: type: object # NOTE: Not guaranteed to be present: eth_hash, to, amount. - required: [round, index, timestamp, hash, sender_0, nonce_0, fee, gas_limit, gas_used, size, method, body, success] + required: [round, index, timestamp, hash, sender_0, nonce_0, fee, gas_limit, gas_used, size, method, body] properties: round: type: integer @@ -2053,7 +2053,14 @@ components: description: The total byte size of the transaction. method: type: string - description: The method that was called. + description: | + The method that was called. Defined by the runtime. In theory, this could be any string as the runtimes evolve. + In practice, the indexer currently expects only the following methods: + - "accounts.Transfer" + - "consensus.Deposit" + - "consensus.Withdraw" + - "evm.Create" + - "evm.Call" example: "evm.Call" body: type: object @@ -2084,7 +2091,9 @@ components: example: "100000001666393459" success: type: boolean - description: Whether this transaction successfully executed. + description: | + Whether this transaction successfully executed. + Can be absent (meaning "unknown") for confidential runtimes. error: $ref: '#/components/schemas/TxError' description: Error details of a failed transaction. diff --git a/api/v1/logic.go b/api/v1/logic.go deleted file mode 100644 index 466a0cecb..000000000 --- a/api/v1/logic.go +++ /dev/null @@ -1,160 +0,0 @@ -package v1 - -import ( - "fmt" - - "github.com/oasisprotocol/oasis-core/go/common/cbor" - "github.com/oasisprotocol/oasis-sdk/client-sdk/go/modules/accounts" - "github.com/oasisprotocol/oasis-sdk/client-sdk/go/modules/consensusaccounts" - "github.com/oasisprotocol/oasis-sdk/client-sdk/go/modules/evm" - "github.com/oasisprotocol/oasis-sdk/client-sdk/go/types" - - "github.com/oasisprotocol/oasis-indexer/analyzer/runtime" - uncategorized "github.com/oasisprotocol/oasis-indexer/analyzer/uncategorized" - apiTypes "github.com/oasisprotocol/oasis-indexer/api/v1/types" - "github.com/oasisprotocol/oasis-indexer/storage/client" -) - -func renderRuntimeTransaction(storageTransaction client.RuntimeTransaction) (apiTypes.RuntimeTransaction, error) { - var utx types.UnverifiedTransaction - if err := cbor.Unmarshal(storageTransaction.Raw, &utx); err != nil { - return apiTypes.RuntimeTransaction{}, fmt.Errorf("utx unmarshal: %w", err) - } - tx, err := uncategorized.OpenUtxNoVerify(&utx) - if err != nil { - return apiTypes.RuntimeTransaction{}, fmt.Errorf("utx open no verify: %w", err) - } - sender0Addr, err := uncategorized.StringifyAddressSpec(&tx.AuthInfo.SignerInfo[0].AddressSpec) - if err != nil { - return apiTypes.RuntimeTransaction{}, fmt.Errorf("signer 0: %w", err) - } - sender0 := string(sender0Addr) - sender0Eth := uncategorized.EthAddrReference(storageTransaction.AddressPreimage[sender0]) - var cr types.CallResult - if err = cbor.Unmarshal(storageTransaction.ResultRaw, &cr); err != nil { - return apiTypes.RuntimeTransaction{}, fmt.Errorf("result unmarshal: %w", err) - } - var body map[string]interface{} - if err = cbor.Unmarshal(tx.Call.Body, &body); err != nil { - return apiTypes.RuntimeTransaction{}, fmt.Errorf("body unmarshal: %w", err) - } - apiTransaction := apiTypes.RuntimeTransaction{ - Round: storageTransaction.Round, - Index: storageTransaction.Index, - Hash: storageTransaction.Hash, - EthHash: storageTransaction.EthHash, - GasUsed: storageTransaction.GasUsed, - Size: storageTransaction.Size, - Timestamp: storageTransaction.Timestamp, - Sender0: sender0, - Sender0Eth: sender0Eth, - Nonce0: tx.AuthInfo.SignerInfo[0].Nonce, - Fee: tx.AuthInfo.Fee.Amount.Amount.String(), - GasLimit: tx.AuthInfo.Fee.Gas, - Method: tx.Call.Method, - Body: body, - Success: cr.IsSuccess(), - } - if !cr.IsSuccess() { - // `module` should be present but message may be null - // https://github.com/oasisprotocol/oasis-sdk/blob/fb741678585c04fdb413441f2bfba18aafbf98f3/client-sdk/go/types/transaction.go#L488-L492 - apiTransaction.Error = &apiTypes.TxError{ - Code: cr.Failed.Code, - Module: &cr.Failed.Module, - } - if len(cr.Failed.Message) > 0 { - apiTransaction.Error.Message = &cr.Failed.Message - } - } - if err = runtime.VisitCall(&tx.Call, &cr, &runtime.CallHandler{ - AccountsTransfer: func(body *accounts.Transfer) error { - toAddr, err2 := uncategorized.StringifySdkAddress(&body.To) - if err2 != nil { - return fmt.Errorf("to: %w", err2) - } - to := string(toAddr) - apiTransaction.To = &to - amount, err2 := uncategorized.StringifyNativeDenomination(&body.Amount) - if err2 != nil { - return fmt.Errorf("amount: %w", err2) - } - apiTransaction.Amount = &amount - return nil - }, - ConsensusAccountsDeposit: func(body *consensusaccounts.Deposit) error { - if body.To != nil { - toAddr, err2 := uncategorized.StringifySdkAddress(body.To) - if err2 != nil { - return fmt.Errorf("to: %w", err2) - } - to := string(toAddr) - toEth := uncategorized.EthAddrReference(storageTransaction.AddressPreimage[to]) - apiTransaction.To = &to - apiTransaction.ToEth = toEth - } else { - apiTransaction.To = &sender0 - apiTransaction.ToEth = sender0Eth - } - amount, err2 := uncategorized.StringifyNativeDenomination(&body.Amount) - if err2 != nil { - return fmt.Errorf("amount: %w", err2) - } - apiTransaction.Amount = &amount - return nil - }, - ConsensusAccountsWithdraw: func(body *consensusaccounts.Withdraw) error { - if body.To != nil { - toAddr, err2 := uncategorized.StringifySdkAddress(body.To) - if err2 != nil { - return fmt.Errorf("to: %w", err2) - } - to := string(toAddr) - toEth := uncategorized.EthAddrReference(storageTransaction.AddressPreimage[to]) - // Beware, this is the address of an account in the consensus - // layer, not an account in the runtime indicated in this API - // request. - apiTransaction.To = &to - apiTransaction.ToEth = toEth - } else { - apiTransaction.To = &sender0 - apiTransaction.ToEth = sender0Eth - } - // todo: ensure native denomination? - amount := body.Amount.Amount.String() - apiTransaction.Amount = &amount - return nil - }, - EVMCreate: func(body *evm.Create, ok *[]byte) error { - if !cr.IsUnknown() && cr.IsSuccess() && len(*ok) == 32 { - // todo: is this rigorous enough? - toAddr, err2 := uncategorized.StringifyEthAddress(uncategorized.SliceEthAddress(*ok)) - if err2 != nil { - return fmt.Errorf("created contract: %w", err2) - } - to := string(toAddr) - toEth := uncategorized.EthAddrReference(storageTransaction.AddressPreimage[to]) - apiTransaction.To = &to - apiTransaction.ToEth = toEth - } - amount := uncategorized.StringifyBytes(body.Value) - apiTransaction.Amount = &amount - return nil - }, - EVMCall: func(body *evm.Call, ok *[]byte) error { - toAddr, err2 := uncategorized.StringifyEthAddress(body.Address) - if err2 != nil { - return fmt.Errorf("to: %w", err2) - } - to := string(toAddr) - toEth := uncategorized.EthAddrReference(storageTransaction.AddressPreimage[to]) - apiTransaction.To = &to - apiTransaction.ToEth = toEth - amount := uncategorized.StringifyBytes(body.Value) - apiTransaction.Amount = &amount - return nil - }, - }); err != nil { - return apiTypes.RuntimeTransaction{}, err - } - return apiTransaction, nil -} diff --git a/api/v1/strict_server.go b/api/v1/strict_server.go index bd7551287..1463a54b0 100644 --- a/api/v1/strict_server.go +++ b/api/v1/strict_server.go @@ -252,53 +252,22 @@ func (srv *StrictServerImpl) GetRuntimeEvmTokens(ctx context.Context, request ap } func (srv *StrictServerImpl) GetRuntimeTransactions(ctx context.Context, request apiTypes.GetRuntimeTransactionsRequestObject) (apiTypes.GetRuntimeTransactionsResponseObject, error) { - storageTransactions, err := srv.dbClient.RuntimeTransactions(ctx, request.Params, nil) + transactions, err := srv.dbClient.RuntimeTransactions(ctx, request.Params, nil) if err != nil { return nil, err } - - // Perform additional tx body parsing on the fly; DB stores only partially-parsed txs. - apiTransactions := apiTypes.RuntimeTransactionList{ - Transactions: []apiTypes.RuntimeTransaction{}, - TotalCount: storageTransactions.TotalCount, - IsTotalCountClipped: storageTransactions.IsTotalCountClipped, - } - for _, storageTransaction := range storageTransactions.Transactions { - apiTransaction, err2 := renderRuntimeTransaction(storageTransaction) - if err2 != nil { - return nil, fmt.Errorf("round %d tx %d: %w", storageTransaction.Round, storageTransaction.Index, err2) - } - apiTransactions.Transactions = append(apiTransactions.Transactions, apiTransaction) - } - - return apiTypes.GetRuntimeTransactions200JSONResponse(apiTransactions), nil + return apiTypes.GetRuntimeTransactions200JSONResponse(*transactions), nil } func (srv *StrictServerImpl) GetRuntimeTransactionsTxHash(ctx context.Context, request apiTypes.GetRuntimeTransactionsTxHashRequestObject) (apiTypes.GetRuntimeTransactionsTxHashResponseObject, error) { - storageTransactions, err := srv.dbClient.RuntimeTransactions(ctx, apiTypes.GetRuntimeTransactionsParams{}, &request.TxHash) + transactions, err := srv.dbClient.RuntimeTransactions(ctx, apiTypes.GetRuntimeTransactionsParams{}, &request.TxHash) if err != nil { return nil, err } - - if len(storageTransactions.Transactions) == 0 { + if len(transactions.Transactions) == 0 { return apiTypes.GetRuntimeTransactionsTxHash404JSONResponse{}, nil } - - // Perform additional tx body parsing on the fly; DB stores only partially-parsed txs. - apiTransactions := apiTypes.RuntimeTransactionList{ - Transactions: []apiTypes.RuntimeTransaction{}, - TotalCount: storageTransactions.TotalCount, - IsTotalCountClipped: storageTransactions.IsTotalCountClipped, - } - for _, storageTransaction := range storageTransactions.Transactions { - apiTransaction, err2 := renderRuntimeTransaction(storageTransaction) - if err2 != nil { - return nil, fmt.Errorf("round %d tx %d: %w", storageTransaction.Round, storageTransaction.Index, err2) - } - apiTransactions.Transactions = append(apiTransactions.Transactions, apiTransaction) - } - - return apiTypes.GetRuntimeTransactionsTxHash200JSONResponse(apiTransactions), nil + return apiTypes.GetRuntimeTransactionsTxHash200JSONResponse(*transactions), nil } func (srv *StrictServerImpl) GetRuntimeEvents(ctx context.Context, request apiTypes.GetRuntimeEventsRequestObject) (apiTypes.GetRuntimeEventsResponseObject, error) { diff --git a/common/types.go b/common/types.go index d47736d61..dfc4007c2 100644 --- a/common/types.go +++ b/common/types.go @@ -84,6 +84,10 @@ func NumericToBigInt(n pgtype.Numeric) (BigInt, error) { return BigInt{Int: *big0}, nil } +func Ptr[T any](v T) *T { + return &v +} + // Key used to set values in a web request context. API uses this to set // values, backend uses this to retrieve values. type ContextKey string diff --git a/storage/client/client.go b/storage/client/client.go index 1d01556ad..ed6c802a2 100644 --- a/storage/client/client.go +++ b/storage/client/client.go @@ -7,6 +7,7 @@ import ( "time" "github.com/dgraph-io/ristretto" + ethCommon "github.com/ethereum/go-ethereum/common" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -1054,21 +1055,44 @@ func (c *StorageClient) RuntimeTransactions(ctx context.Context, p apiTypes.GetR IsTotalCountClipped: res.isTotalCountClipped, } for res.rows.Next() { - var t RuntimeTransaction + t := RuntimeTransaction{ + Error: &TxError{}, + } if err := res.rows.Scan( &t.Round, &t.Index, + &t.Timestamp, &t.Hash, &t.EthHash, + &t.Sender0, + &t.Sender0Eth, + &t.Nonce0, + &t.Fee, + &t.GasLimit, &t.GasUsed, &t.Size, - &t.Timestamp, - &t.Raw, - &t.ResultRaw, - &t.AddressPreimage, + &t.Method, + &t.Body, + &t.To, + &t.ToEth, + &t.Amount, + &t.Success, + &t.Error.Module, + &t.Error.Code, + &t.Error.Message, ); err != nil { return nil, wrapError(err) } + if t.Success != nil && *t.Success { + t.Error = nil + } + // Fancy-format eth addresses: Apply checksum capitalization, prepend 0x. + if t.Sender0Eth != nil { + *t.Sender0Eth = ethCommon.HexToAddress(*t.Sender0Eth).Hex() + } + if t.ToEth != nil { + *t.ToEth = ethCommon.HexToAddress(*t.ToEth).Hex() + } ts.Transactions = append(ts.Transactions, t) } diff --git a/storage/client/queries/queries.go b/storage/client/queries/queries.go index cfce579f9..b171c2e56 100644 --- a/storage/client/queries/queries.go +++ b/storage/client/queries/queries.go @@ -313,33 +313,46 @@ const ( SELECT txs.round, txs.tx_index, + txs.timestamp, txs.tx_hash, txs.tx_eth_hash, + signer0.signer_address AS sender0, + encode(signer_eth.address_data, 'hex') AS sender0_eth, + signer0.nonce AS nonce0, + txs.fee, + txs.gas_limit, txs.gas_used, txs.size, - txs.timestamp, - txs.raw, - txs.result_raw, - ( - SELECT - json_object_agg(pre.address, encode(pre.address_data, 'hex')) - FROM chain.runtime_related_transactions AS rel - JOIN chain.address_preimages AS pre ON rel.account_address = pre.address - WHERE txs.runtime = rel.runtime - AND txs.round = rel.tx_round - AND txs.tx_index = rel.tx_index - ) AS eth_addr_lookup + txs.method, + txs.body, + txs.to, + encode(to_eth.address_data, 'hex') AS to_eth, + txs.amount, + txs.success, + txs.error_module, + txs.error_code, + txs.error_message FROM chain.runtime_transactions AS txs - LEFT JOIN chain.runtime_related_transactions AS rel ON txs.round = rel.tx_round - AND txs.tx_index = rel.tx_index - AND txs.runtime = rel.runtime + LEFT JOIN chain.runtime_transaction_signers AS signer0 USING (runtime, round, tx_index) + LEFT JOIN chain.address_preimages AS signer_eth ON + (signer0.signer_address = signer_eth.address) AND + (signer_eth.context_identifier = 'oasis-runtime-sdk/address: secp256k1eth') AND (signer_eth.context_version = 0) + LEFT JOIN chain.address_preimages AS to_eth ON + (txs.to = to_eth.address) AND + (to_eth.context_identifier = 'oasis-runtime-sdk/address: secp256k1eth') AND (to_eth.context_version = 0) + LEFT JOIN chain.runtime_related_transactions AS rel ON + (txs.round = rel.tx_round) AND + (txs.tx_index = rel.tx_index) AND + (txs.runtime = rel.runtime) AND -- When related_address ($4) is NULL and hence we do no filtering on it, avoid the join altogether. -- Otherwise, every tx will be returned as many times as there are related addresses for it. - AND $4::text IS NOT NULL - WHERE (txs.runtime = $1) AND - ($2::bigint IS NULL OR txs.round = $2::bigint) AND - ($3::text IS NULL OR txs.tx_hash = $3::text OR txs.tx_eth_hash = $3::text) AND - ($4::text IS NULL OR rel.account_address = $4::text) + ($4::text IS NOT NULL) + WHERE + (txs.runtime = $1) AND + (signer0.signer_index = 0) AND + ($2::bigint IS NULL OR txs.round = $2::bigint) AND + ($3::text IS NULL OR txs.tx_hash = $3::text OR txs.tx_eth_hash = $3::text) AND + ($4::text IS NULL OR rel.account_address = $4::text) ORDER BY txs.round DESC, txs.tx_index DESC LIMIT $5::bigint OFFSET $6::bigint @@ -355,7 +368,7 @@ const ( ($5::text IS NULL OR type = $5::text) AND ($6::text IS NULL OR evm_log_signature = $6::text) AND ($7::text IS NULL OR related_accounts @> ARRAY[$7::text]) - ORDER BY round DESC, tx_index + ORDER BY round DESC, tx_index, type, body::text LIMIT $8::bigint OFFSET $9::bigint` diff --git a/storage/client/types.go b/storage/client/types.go index fced05b18..b496d2890 100644 --- a/storage/client/types.go +++ b/storage/client/types.go @@ -2,8 +2,6 @@ package client import ( - "time" - api "github.com/oasisprotocol/oasis-indexer/api/v1/types" ) @@ -98,30 +96,11 @@ type RuntimeBlockList = api.RuntimeBlockList type RuntimeBlock = api.RuntimeBlock // RuntimeTransactionList is the storage response for RuntimeTransactions. -type RuntimeTransactionList struct { - Transactions []RuntimeTransaction `json:"transactions"` - TotalCount uint64 - IsTotalCountClipped bool -} - -// RuntimeTransaction is the storage response for RuntimeTransaction. -// It differs from what the API returns; the DB stores a less-parsed -// version of the transaction, and we finish parsing on the fly, as we -// return the tx. -type RuntimeTransaction struct { - Round int64 - Index int64 - Hash string - EthHash *string - GasUsed uint64 - Size int32 - Sender0 *string - Sender0Eth *string - Timestamp time.Time - Raw []byte - ResultRaw []byte - AddressPreimage map[string]string -} +type ( + RuntimeTransactionList = api.RuntimeTransactionList + RuntimeTransaction = api.RuntimeTransaction + TxError = api.TxError +) // RuntimeEventList is the storage response for RuntimeEvents. type RuntimeEventList = api.RuntimeEventList diff --git a/storage/migrations/02_runtimes.up.sql b/storage/migrations/02_runtimes.up.sql index dd8603371..d37cb59b9 100644 --- a/storage/migrations/02_runtimes.up.sql +++ b/storage/migrations/02_runtimes.up.sql @@ -34,19 +34,30 @@ CREATE TABLE chain.runtime_transactions round UINT63 NOT NULL, FOREIGN KEY (runtime, round) REFERENCES chain.runtime_blocks DEFERRABLE INITIALLY DEFERRED, tx_index UINT31 NOT NULL, + PRIMARY KEY (runtime, round, tx_index), + timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + tx_hash HEX64 NOT NULL, tx_eth_hash HEX64, - gas_used UINT63 NOT NULL, + -- NOTE: The signer(s) and their nonce(s) are stored separately in runtime_transaction_signers. + + fee UINT_NUMERIC NOT NULL, + gas_limit UINT63 NOT NULL, + gas_used UINT63 NOT NULL, + size UINT31 NOT NULL, - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - -- raw is cbor(UnverifiedTransaction). If you're unable to get a copy of the - -- transaction from the node itself, parse from here. Remove this if we - -- later store sufficiently detailed data in other columns or if we turn out - -- to be able to get a copy of the transaction elsewhere. - raw BYTEA NOT NULL, - -- result_raw is cbor(CallResult). - result_raw BYTEA NOT NULL, - PRIMARY KEY (runtime, round, tx_index) + + -- Transaction contents. + method TEXT NOT NULL, -- accounts.Transter, consensus.Deposit, consensus.Withdraw, evm.Create, evm.Call + body JSON NOT NULL, -- For EVM txs, the EVM method and args are encoded in here. + "to" oasis_addr, -- Exact semantics depend on method. Extracted from body; for convenience only. + amount UINT_NUMERIC, -- Exact semantics depend on method. Extracted from body; for convenience only. + + -- Error information. + success BOOLEAN, -- NULL means success is unknown (can happen in confidential runtimes) + error_module TEXT, + error_code UINT63, + error_message TEXT ); CREATE INDEX ix_runtime_transactions_tx_hash ON chain.runtime_transactions USING hash (tx_hash); CREATE INDEX ix_runtime_transactions_tx_eth_hash ON chain.runtime_transactions USING hash (tx_eth_hash); @@ -62,7 +73,7 @@ CREATE TABLE chain.runtime_transaction_signers -- from multisig accounts). signer_index UINT31 NOT NULL, signer_address oasis_addr NOT NULL, - nonce UINT31 NOT NULL, + nonce UINT63 NOT NULL, PRIMARY KEY (runtime, round, tx_index, signer_index), FOREIGN KEY (runtime, round, tx_index) REFERENCES chain.runtime_transactions(runtime, round, tx_index) DEFERRABLE INITIALLY DEFERRED ); diff --git a/tests/e2e_regression/e2e_config.yaml b/tests/e2e_regression/e2e_config.yaml new file mode 100644 index 000000000..23ea0944f --- /dev/null +++ b/tests/e2e_regression/e2e_config.yaml @@ -0,0 +1,40 @@ +# Indexer configuration for non-synthetic e2e regression tests implemented in this directory. + +analysis: + node: + chain_id: oasis-3 + rpc: unix:/tmp/node.sock + chaincontext: b11b369e0da5bb230b220127f5e7b242d385ef8c6f54906243f30af63c815535 + fast_startup: true + analyzers: + consensus: + from: 8_048_956 # Damask genesis + to: 8_049_056 # 100 blocks; fast enough for early testing + emerald: + from: 1_003_298 # round at Damask genesis + to: 1_003_398 # 100 blocks; fast enough for early testing + evm_tokens: {} + aggregate_stats: + tx_volume_interval: 5m + metadata_registry: + interval: 5m + storage: + endpoint: postgresql://rwuser:password@localhost:5432/indexer?sslmode=disable + backend: postgres + DANGER__WIPE_STORAGE_ON_STARTUP: true + migrations: file://storage/migrations + +server: + chain_id: oasis-3 + chain_name: mainnet + endpoint: localhost:8008 + storage: + endpoint: postgresql://api:password@localhost:5432/indexer?sslmode=disable + backend: postgres + +log: + level: debug + format: json + +metrics: + pull_endpoint: localhost:8009 diff --git a/tests/e2e_regression/reindex_and_run.sh b/tests/e2e_regression/reindex_and_run.sh index 0152a97aa..d78bf5aab 100755 --- a/tests/e2e_regression/reindex_and_run.sh +++ b/tests/e2e_regression/reindex_and_run.sh @@ -4,59 +4,19 @@ # before running the e2e regression tests. set -euo pipefail - -cat >/tmp/e2e_config.yaml </dev/null 2>&1 && pwd )" # Kill background processes on exit. (In our case the indexer API server.) trap 'trap - SIGTERM && kill -- -$$' SIGINT SIGTERM EXIT make oasis-indexer -./oasis-indexer --config=/tmp/e2e_config.yaml analyze | tee /tmp/analyze.out & +./oasis-indexer --config="${SCRIPT_DIR}/e2e_config.yaml" analyze | tee /tmp/analyze.out & analyzer_pid=$! # Count how many block analyzers are enabled in the config. n_block_analyzers=0 for analyzer in consensus emerald sapphire cipher; do - if grep -qE "^ *${analyzer}:" /tmp/e2e_config.yaml; then + if grep -qE "^ *${analyzer}:" "${SCRIPT_DIR}/e2e_config.yaml"; then n_block_analyzers=$((n_block_analyzers + 1)) fi done @@ -70,10 +30,5 @@ done sleep 2 # Give evm_tokens analyzer (and other non-block analyzers) a chance to finish. kill $analyzer_pid -./oasis-indexer --config=/tmp/e2e_config.yaml serve & -while ! curl --silent localhost:8008/v1/ >/dev/null; do - echo "Waiting for API server to start..." - sleep 1 -done tests/e2e_regression/run.sh diff --git a/tests/e2e_regression/run.sh b/tests/e2e_regression/run.sh index 27b56804f..50f250bee 100755 --- a/tests/e2e_regression/run.sh +++ b/tests/e2e_regression/run.sh @@ -68,8 +68,19 @@ testCases=( ) nCases=${#testCases[@]} -seen=() +# Kill background processes on exit. (In our case the indexer API server.) +trap 'trap - SIGTERM && kill -- -$$' SIGINT SIGTERM EXIT + +# Start the API server. +make oasis-indexer +./oasis-indexer --config="${SCRIPT_DIR}/e2e_config.yaml" serve & +while ! curl --silent localhost:8008/v1/ >/dev/null; do + echo "Waiting for API server to start..." + sleep 1 +done +# Run the test cases. +seen=() for (( i=0; i