Skip to content

Commit

Permalink
update indexer to read consensus blocks from file
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrew7234 committed Mar 20, 2023
1 parent 37480f0 commit c03bb05
Show file tree
Hide file tree
Showing 4 changed files with 182 additions and 21 deletions.
4 changes: 2 additions & 2 deletions storage/oasis/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ func (cf *ClientFactory) Consensus() (*ConsensusClient, error) {
signature.SetChainContext(cf.network.ChainContext)

c := &ConsensusClient{
nodeApi: nodeApi,
network: cf.network,
NodeApi: nodeApi,
Network: cf.network,
}

return c, nil
Expand Down
38 changes: 19 additions & 19 deletions storage/oasis/consensus.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,26 @@ import (
registryAPI "github.com/oasisprotocol/oasis-core/go/registry/api"
schedulerAPI "github.com/oasisprotocol/oasis-core/go/scheduler/api"
stakingAPI "github.com/oasisprotocol/oasis-core/go/staking/api"
"github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi"
config "github.com/oasisprotocol/oasis-sdk/client-sdk/go/config"

"github.com/oasisprotocol/oasis-indexer/storage"
"github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi"
)

// ConsensusClient is a client to the consensus backends.
type ConsensusClient struct {
nodeApi nodeapi.ConsensusApiLite
network *config.Network
NodeApi nodeapi.ConsensusApiLite
Network *config.Network
}

// GenesisDocument returns the original genesis document.
func (cc *ConsensusClient) GenesisDocument(ctx context.Context) (*genesisAPI.Document, error) {
return cc.nodeApi.GetGenesisDocument(ctx)
return cc.NodeApi.GetGenesisDocument(ctx)
}

// GenesisDocumentAtHeight returns the genesis document at the provided height.
func (cc *ConsensusClient) GenesisDocumentAtHeight(ctx context.Context, height int64) (*genesisAPI.Document, error) {
return cc.nodeApi.StateToGenesis(ctx, height)
return cc.NodeApi.StateToGenesis(ctx, height)
}

// Name returns the name of the client, for the ConsensusSourceStorage interface.
Expand All @@ -42,7 +42,7 @@ func (cc *ConsensusClient) Name() string {

// GetEpoch returns the epoch number at the specified block height.
func (cc *ConsensusClient) GetEpoch(ctx context.Context, height int64) (beaconAPI.EpochTime, error) {
return cc.nodeApi.GetEpoch(ctx, height)
return cc.NodeApi.GetEpoch(ctx, height)
}

// AllData returns all relevant data related to the given height.
Expand Down Expand Up @@ -90,7 +90,7 @@ func (cc *ConsensusClient) AllData(ctx context.Context, height int64) (*storage.

// BlockData retrieves data about a consensus block at the provided block height.
func (cc *ConsensusClient) BlockData(ctx context.Context, height int64) (*storage.ConsensusBlockData, error) {
block, err := cc.nodeApi.GetBlock(ctx, height)
block, err := cc.NodeApi.GetBlock(ctx, height)
if err != nil {
return nil, err
}
Expand All @@ -100,7 +100,7 @@ func (cc *ConsensusClient) BlockData(ctx context.Context, height int64) (*storag
return nil, err
}

transactionsWithResults, err := cc.nodeApi.GetTransactionsWithResults(ctx, height)
transactionsWithResults, err := cc.NodeApi.GetTransactionsWithResults(ctx, height)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -146,7 +146,7 @@ func (cc *ConsensusClient) BeaconData(ctx context.Context, height int64) (*stora

// RegistryData retrieves registry events at the provided block height.
func (cc *ConsensusClient) RegistryData(ctx context.Context, height int64) (*storage.RegistryData, error) {
events, err := cc.nodeApi.RegistryEvents(ctx, height)
events, err := cc.NodeApi.RegistryEvents(ctx, height)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -181,7 +181,7 @@ func (cc *ConsensusClient) RegistryData(ctx context.Context, height int64) (*sto

// StakingData retrieves staking events at the provided block height.
func (cc *ConsensusClient) StakingData(ctx context.Context, height int64) (*storage.StakingData, error) {
events, err := cc.nodeApi.StakingEvents(ctx, height)
events, err := cc.NodeApi.StakingEvents(ctx, height)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -222,20 +222,20 @@ func (cc *ConsensusClient) StakingData(ctx context.Context, height int64) (*stor

// SchedulerData retrieves validators and runtime committees at the provided block height.
func (cc *ConsensusClient) SchedulerData(ctx context.Context, height int64) (*storage.SchedulerData, error) {
validators, err := cc.nodeApi.GetValidators(ctx, height)
validators, err := cc.NodeApi.GetValidators(ctx, height)
if err != nil {
return nil, err
}

committees := make(map[common.Namespace][]*schedulerAPI.Committee, len(cc.network.ParaTimes.All))
committees := make(map[common.Namespace][]*schedulerAPI.Committee, len(cc.Network.ParaTimes.All))

for name := range cc.network.ParaTimes.All {
for name := range cc.Network.ParaTimes.All {
var runtimeID common.Namespace
if err := runtimeID.UnmarshalHex(cc.network.ParaTimes.All[name].ID); err != nil {
if err := runtimeID.UnmarshalHex(cc.Network.ParaTimes.All[name].ID); err != nil {
return nil, err
}

consensusCommittees, err := cc.nodeApi.GetCommittees(ctx, height, runtimeID)
consensusCommittees, err := cc.NodeApi.GetCommittees(ctx, height, runtimeID)
if err != nil {
return nil, err
}
Expand All @@ -251,7 +251,7 @@ func (cc *ConsensusClient) SchedulerData(ctx context.Context, height int64) (*st

// GovernanceData retrieves governance events at the provided block height.
func (cc *ConsensusClient) GovernanceData(ctx context.Context, height int64) (*storage.GovernanceData, error) {
events, err := cc.nodeApi.GovernanceEvents(ctx, height)
events, err := cc.NodeApi.GovernanceEvents(ctx, height)
if err != nil {
return nil, err
}
Expand All @@ -264,15 +264,15 @@ func (cc *ConsensusClient) GovernanceData(ctx context.Context, height int64) (*s
for _, event := range events {
switch e := event; {
case e.ProposalSubmitted != nil:
proposal, err := cc.nodeApi.GetProposal(ctx, height, event.ProposalSubmitted.ID)
proposal, err := cc.NodeApi.GetProposal(ctx, height, event.ProposalSubmitted.ID)
if err != nil {
return nil, err
}
submissions = append(submissions, proposal)
case e.ProposalExecuted != nil:
executions = append(executions, event.ProposalExecuted)
case e.ProposalFinalized != nil:
proposal, err := cc.nodeApi.GetProposal(ctx, height, event.ProposalFinalized.ID)
proposal, err := cc.NodeApi.GetProposal(ctx, height, event.ProposalFinalized.ID)
if err != nil {
return nil, err
}
Expand All @@ -293,7 +293,7 @@ func (cc *ConsensusClient) GovernanceData(ctx context.Context, height int64) (*s

// RootHashData retrieves roothash events at the provided block height.
func (cc *ConsensusClient) RootHashData(ctx context.Context, height int64) (*storage.RootHashData, error) {
events, err := cc.nodeApi.RoothashEvents(ctx, height)
events, err := cc.NodeApi.RoothashEvents(ctx, height)
if err != nil {
return nil, err
}
Expand Down
12 changes: 12 additions & 0 deletions storage/oasis/nodeapi/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,15 @@ type ConsensusApiLite interface {
GetCommittees(ctx context.Context, height int64, runtimeID common.Namespace) ([]*scheduler.Committee, error)
GetProposal(ctx context.Context, height int64, proposalID uint64) (*governance.Proposal, error)
}

type ConsensusApiLiteBlockData struct {
Block consensus.Block `json:"block"`
TransactionsWithResults consensus.TransactionsWithResults `json:"transactions_with_results"`
Epoch beacon.EpochTime `json:"epoch"`
RegistryEvents []*registry.Event `json:"registry_events"`
StakingEvents []*staking.Event `json:"staking_events"`
GovernanceEvents []*governance.Event `json:"governance_events"`
RootHashEvents []*roothash.Event `json:"roothash_events"`
Validators []*scheduler.Validator `json:"validators"`
Committees map[common.Namespace][]*scheduler.Committee `json:"committees"`
}
149 changes: 149 additions & 0 deletions storage/oasis/nodeapi/file/node.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package file

import (
"context"
"encoding/json"
"os"

"github.com/oasisprotocol/oasis-core/go/common"
consensus "github.com/oasisprotocol/oasis-core/go/consensus/api"
"github.com/oasisprotocol/oasis-indexer/storage/oasis/nodeapi"

// indexer-internal data types.
beacon "github.com/oasisprotocol/oasis-core/go/beacon/api"
genesis "github.com/oasisprotocol/oasis-core/go/genesis/api"
governance "github.com/oasisprotocol/oasis-core/go/governance/api"
registry "github.com/oasisprotocol/oasis-core/go/registry/api"
roothash "github.com/oasisprotocol/oasis-core/go/roothash/api"
scheduler "github.com/oasisprotocol/oasis-core/go/scheduler/api"
staking "github.com/oasisprotocol/oasis-core/go/staking/api"
)

// FileConsensusApiLite provides access to the consensus API of an Oasis node.
// Since FileConsensusApiLite is backed by a file containing the cached responses
// to `ConsensusApiLite` calls, this data is inherently compatible with the
// current indexer and can thus handle heights from both Cobalt/Damask.
type FileConsensusApiLite struct {
currHeight int64
currData *nodeapi.ConsensusApiLiteBlockData // cache the current height's data to speed sequential reads.
genesisDocument *genesis.Document
filename string
reader *json.Decoder
}

var _ nodeapi.ConsensusApiLite = (*FileConsensusApiLite)(nil)

func NewFileConsensusApiLite(filename string) (*FileConsensusApiLite, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
reader := json.NewDecoder(f)
var genesisDocument genesis.Document
if err := reader.Decode(genesisDocument); err != nil {
return nil, err
}

return &FileConsensusApiLite{
currHeight: 0,
genesisDocument: &genesisDocument,
filename: filename,
reader: reader,
}, nil
}

func (c *FileConsensusApiLite) jumpToHeight(height int64) error {
if height == c.currHeight {
return nil
}
if height < c.currHeight {
f, err := os.Open(c.filename)
if err != nil {
return err
}
c.reader = json.NewDecoder(f)
if height == 0 {
return nil
}

var v interface{}
c.reader.Decode(v) // skip genesis document
c.reader.Decode(c.currData)
c.currHeight = c.currData.Block.Height
}

for c.currHeight < height && c.reader.More() {
if err := c.reader.Decode(c.currData); err != nil { // todo: use file reader readline() instead to avoid serializing into c.currData
return err
}
c.currHeight = c.currHeight + 1
}

return nil
}

func (c *FileConsensusApiLite) GetGenesisDocument(ctx context.Context) (*genesis.Document, error) {
return c.genesisDocument, nil
}

func (c *FileConsensusApiLite) StateToGenesis(ctx context.Context, height int64) (*genesis.Document, error) {
panic("not implemented") // Only used by the statecheck as of 16/03/2023
}

func (c *FileConsensusApiLite) GetBlock(ctx context.Context, height int64) (*consensus.Block, error) {
c.jumpToHeight(height)

return &c.currData.Block, nil
}

func (c *FileConsensusApiLite) GetTransactionsWithResults(ctx context.Context, height int64) (*consensus.TransactionsWithResults, error) {
c.jumpToHeight(height)

return &c.currData.TransactionsWithResults, nil
}

func (c *FileConsensusApiLite) GetEpoch(ctx context.Context, height int64) (beacon.EpochTime, error) {
c.jumpToHeight(height)

return c.currData.Epoch, nil
}

func (c *FileConsensusApiLite) RegistryEvents(ctx context.Context, height int64) ([]*registry.Event, error) {
c.jumpToHeight(height)

return c.currData.RegistryEvents, nil
}

func (c *FileConsensusApiLite) StakingEvents(ctx context.Context, height int64) ([]*staking.Event, error) {
c.jumpToHeight(height)

return c.currData.StakingEvents, nil
}

func (c *FileConsensusApiLite) GovernanceEvents(ctx context.Context, height int64) ([]*governance.Event, error) {
c.jumpToHeight(height)

return c.currData.GovernanceEvents, nil
}

func (c *FileConsensusApiLite) RoothashEvents(ctx context.Context, height int64) ([]*roothash.Event, error) {
c.jumpToHeight(height)

return c.currData.RootHashEvents, nil
}

func (c *FileConsensusApiLite) GetValidators(ctx context.Context, height int64) ([]*scheduler.Validator, error) {
c.jumpToHeight(height)

return c.currData.Validators, nil
}

func (c *FileConsensusApiLite) GetCommittees(ctx context.Context, height int64, runtimeID common.Namespace) ([]*scheduler.Committee, error) {
c.jumpToHeight(height)

return c.currData.Committees[runtimeID], nil
}

func (c *FileConsensusApiLite) GetProposal(ctx context.Context, height int64, proposalID uint64) (*governance.Proposal, error) {
panic("not implemented")
}

0 comments on commit c03bb05

Please sign in to comment.