Skip to content

Commit

Permalink
Optimization: remove message count field in block stats (#219)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aopoltorzhicky
aopoltorzhicky committed Jun 17, 2024
1 parent cbcbd0e commit df3710b
Show file tree
Hide file tree
Showing 10 changed files with 37 additions and 235 deletions.
4 changes: 0 additions & 4 deletions cmd/api/docs/docs.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 0 additions & 4 deletions cmd/api/docs/swagger.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 0 additions & 3 deletions cmd/api/docs/swagger.yaml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

62 changes: 30 additions & 32 deletions cmd/api/handler/responses/block.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,22 +66,21 @@ func NewBlock(block storage.Block, withStats bool) Block {
}

type BlockStats struct {
TxCount int64 `example:"12" json:"tx_count" swaggertype:"integer"`
EventsCount int64 `example:"18" json:"events_count" swaggertype:"integer"`
BlobsSize int64 `example:"12354" json:"blobs_size" swaggertype:"integer"`
BlobsCount int `example:"100" json:"blobs_count" swaggertype:"integer"`
Fee string `example:"28347628346" json:"fee" swaggertype:"string"`
SupplyChange string `example:"8635234" json:"supply_change" swaggertype:"string"`
InflationRate string `example:"0.0800000" json:"inflation_rate" swaggertype:"string"`
FillRate string `example:"0.0800" json:"fill_rate" swaggertype:"string"`
Rewards string `example:"102102812" json:"rewards" swaggertype:"string"`
Commissions string `example:"123133" json:"commissions" swaggertype:"string"`
BlockTime uint64 `example:"12354" json:"block_time" swaggertype:"integer"`
GasLimit int64 `example:"1234" json:"gas_limit" swaggertype:"integer"`
GasUsed int64 `example:"1234" json:"gas_used" swaggertype:"integer"`
BytesInBlock int64 `example:"1234" json:"bytes_in_block" swaggertype:"integer"`
SquareSize uint64 `example:"16" json:"square_size" swaggertype:"integer"`
MessagesCounts map[types.MsgType]int64 `example:"{MsgPayForBlobs:10,MsgUnjail:1}" json:"messages_counts" swaggertype:"string"`
TxCount int64 `example:"12" json:"tx_count" swaggertype:"integer"`
EventsCount int64 `example:"18" json:"events_count" swaggertype:"integer"`
BlobsSize int64 `example:"12354" json:"blobs_size" swaggertype:"integer"`
BlobsCount int `example:"100" json:"blobs_count" swaggertype:"integer"`
Fee string `example:"28347628346" json:"fee" swaggertype:"string"`
SupplyChange string `example:"8635234" json:"supply_change" swaggertype:"string"`
InflationRate string `example:"0.0800000" json:"inflation_rate" swaggertype:"string"`
FillRate string `example:"0.0800" json:"fill_rate" swaggertype:"string"`
Rewards string `example:"102102812" json:"rewards" swaggertype:"string"`
Commissions string `example:"123133" json:"commissions" swaggertype:"string"`
BlockTime uint64 `example:"12354" json:"block_time" swaggertype:"integer"`
GasLimit int64 `example:"1234" json:"gas_limit" swaggertype:"integer"`
GasUsed int64 `example:"1234" json:"gas_used" swaggertype:"integer"`
BytesInBlock int64 `example:"1234" json:"bytes_in_block" swaggertype:"integer"`
SquareSize uint64 `example:"16" json:"square_size" swaggertype:"integer"`
}

var (
Expand All @@ -90,21 +89,20 @@ var (

func NewBlockStats(stats storage.BlockStats) *BlockStats {
return &BlockStats{
TxCount: stats.TxCount,
EventsCount: stats.EventsCount,
BlobsSize: stats.BlobsSize,
BlobsCount: stats.BlobsCount,
Fee: stats.Fee.String(),
SupplyChange: stats.SupplyChange.String(),
InflationRate: stats.InflationRate.String(),
Commissions: stats.Commissions.String(),
Rewards: stats.Rewards.String(),
BlockTime: stats.BlockTime,
MessagesCounts: stats.MessagesCounts,
GasLimit: stats.GasLimit,
GasUsed: stats.GasUsed,
BytesInBlock: stats.BytesInBlock,
SquareSize: stats.SquareSize,
FillRate: fmt.Sprintf("%.4f", float64(stats.BytesInBlock)/float64(maxSize)),
TxCount: stats.TxCount,
EventsCount: stats.EventsCount,
BlobsSize: stats.BlobsSize,
BlobsCount: stats.BlobsCount,
Fee: stats.Fee.String(),
SupplyChange: stats.SupplyChange.String(),
InflationRate: stats.InflationRate.String(),
Commissions: stats.Commissions.String(),
Rewards: stats.Rewards.String(),
BlockTime: stats.BlockTime,
GasLimit: stats.GasLimit,
GasUsed: stats.GasUsed,
BytesInBlock: stats.BytesInBlock,
SquareSize: stats.SquareSize,
FillRate: fmt.Sprintf("%.4f", float64(stats.BytesInBlock)/float64(maxSize)),
}
}
32 changes: 6 additions & 26 deletions cmd/api/handler/responses/block_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,7 @@ func TestNewBlock(t *testing.T) {
SupplyChange: decimal.NewFromInt(123),
InflationRate: decimal.NewFromFloat(0.08),
Fee: decimal.NewFromInt(125),
MessagesCounts: map[storageTypes.MsgType]int64{
storageTypes.MsgSend: 1,
storageTypes.MsgPayForBlobs: 2,
storageTypes.MsgDelegate: 3,
},
BytesInBlock: 10000,
BytesInBlock: 10000,
},
},
withStats: true,
Expand Down Expand Up @@ -161,13 +156,8 @@ func TestNewBlock(t *testing.T) {
Rewards: "0",
Commissions: "0",
BlockTime: 11000,
MessagesCounts: map[storageTypes.MsgType]int64{
storageTypes.MsgSend: 1,
storageTypes.MsgPayForBlobs: 2,
storageTypes.MsgDelegate: 3,
},
BytesInBlock: 10000,
FillRate: "0.0051",
BytesInBlock: 10000,
FillRate: "0.0051",
},
},
},
Expand Down Expand Up @@ -203,12 +193,7 @@ func TestNewBlockStats(t *testing.T) {
SupplyChange: decimal.NewFromInt(123),
InflationRate: decimal.NewFromFloat(0.08),
Fee: decimal.NewFromInt(125),
MessagesCounts: map[storageTypes.MsgType]int64{
storageTypes.MsgSend: 1,
storageTypes.MsgPayForBlobs: 2,
storageTypes.MsgDelegate: 3,
},
BytesInBlock: 10000,
BytesInBlock: 10000,
},
},
want: &BlockStats{
Expand All @@ -221,13 +206,8 @@ func TestNewBlockStats(t *testing.T) {
Rewards: "0",
Commissions: "0",
BlockTime: 11000,
MessagesCounts: map[storageTypes.MsgType]int64{
storageTypes.MsgSend: 1,
storageTypes.MsgPayForBlobs: 2,
storageTypes.MsgDelegate: 3,
},
BytesInBlock: 10000,
FillRate: "0.0051",
BytesInBlock: 10000,
FillRate: "0.0051",
},
},
}
Expand Down
4 changes: 0 additions & 4 deletions internal/storage/block_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ import (
"context"
"time"

"github.com/celenium-io/celestia-indexer/internal/storage/types"

pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types"
"github.com/shopspring/decimal"
"github.com/uptrace/bun"
Expand Down Expand Up @@ -41,8 +39,6 @@ type BlockStats struct {
Commissions decimal.Decimal `bun:"commissions,type:numeric" comment:"Total commissions per block" stats:"func:min max sum avg"`
BytesInBlock int64 `bun:"bytes_in_block" comment:"Size of all transactions in bytes" stats:"func:min max sum avg"`
SquareSize uint64 `bun:"square_size" comment:"Size of the square after splitting all the block data into shares"`

MessagesCounts map[types.MsgType]int64 `bun:"-"`
}

func (BlockStats) TableName() string {
Expand Down
104 changes: 1 addition & 103 deletions internal/storage/postgres/block.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"time"

"github.com/celenium-io/celestia-indexer/internal/storage"
storageTypes "github.com/celenium-io/celestia-indexer/internal/storage/types"
"github.com/celenium-io/celestia-indexer/pkg/types"
pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types"
"github.com/dipdup-net/go-lib/database"
Expand Down Expand Up @@ -41,11 +40,6 @@ func (b *Blocks) ByHeight(ctx context.Context, height types.Level) (block storag
return
}

type typeCount struct {
Type storageTypes.MsgType `bun:"type"`
Count int64 `bun:"count"`
}

// ByHeightWithStats -
func (b *Blocks) ByHeightWithStats(ctx context.Context, height types.Level) (block storage.Block, err error) {
subQuery := b.DB().NewSelect().Model(&block).
Expand All @@ -62,30 +56,6 @@ func (b *Blocks) ByHeightWithStats(ctx context.Context, height types.Level) (blo
Join("LEFT JOIN validator AS proposer ON (proposer.id = block.proposer_id)").
Scan(ctx, &block)

if err != nil {
return
}

block.Stats.MessagesCounts = make(map[storageTypes.MsgType]int64)

if block.Stats.TxCount > 0 {
var msgsStats []typeCount
err = b.DB().NewSelect().Model((*storage.Message)(nil)).
ColumnExpr("message.type, count(*)").
Where("message.height = ?", height).
Where("message.time = ?", block.Time).
Group("message.type").
Scan(ctx, &msgsStats)

if err != nil {
return
}

for _, stat := range msgsStats {
block.Stats.MessagesCounts[stat.Type] = stat.Count
}
}

return
}

Expand All @@ -105,30 +75,6 @@ func (b *Blocks) ByIdWithRelations(ctx context.Context, id uint64) (block storag
Join("LEFT JOIN validator AS proposer ON (proposer.id = block.proposer_id)").
Scan(ctx, &block)

if err != nil {
return
}

block.Stats.MessagesCounts = make(map[storageTypes.MsgType]int64)

if block.Stats.TxCount > 0 {
var msgsStats []typeCount
err = b.DB().NewSelect().Model((*storage.Message)(nil)).
ColumnExpr("message.type, count(*)").
Where("message.height = ?", block.Height).
Where("message.time = ?", block.Time).
Group("message.type").
Scan(ctx, &msgsStats)

if err != nil {
return
}

for _, stat := range msgsStats {
block.Stats.MessagesCounts[stat.Type] = stat.Count
}
}

return
}

Expand Down Expand Up @@ -163,12 +109,6 @@ func (b *Blocks) ByHash(ctx context.Context, hash []byte) (block storage.Block,
return
}

type listTypeCount struct {
Height types.Level `bun:"height"`
Type storageTypes.MsgType `bun:"type"`
Count int64 `bun:"count"`
}

// ListWithStats -
func (b *Blocks) ListWithStats(ctx context.Context, limit, offset uint64, order sdk.SortOrder) (blocks []*storage.Block, err error) {
subQuery := b.DB().NewSelect().Model(&blocks)
Expand All @@ -185,49 +125,7 @@ func (b *Blocks) ListWithStats(ctx context.Context, limit, offset uint64, order
Join("LEFT JOIN validator as v ON v.id = block.proposer_id")
query = sortScope(query, "block.id", order)

if err = query.Scan(ctx, &blocks); err != nil {
return
}

if len(blocks) == 0 {
return
}

var (
heights = make([]types.Level, len(blocks))
blocksHeightMap = make(map[types.Level]*storage.Block)
startTime = blocks[len(blocks)-1].Time
endTime = blocks[0].Time
)

if order == sdk.SortOrderAsc {
startTime, endTime = endTime, startTime
}

for i := range blocks {
heights[i] = blocks[i].Height
blocksHeightMap[blocks[i].Height] = blocks[i]
blocks[i].Stats.MessagesCounts = make(map[storageTypes.MsgType]int64)
}

var listTypeCounts []listTypeCount
queryMsgsCounts := b.DB().NewSelect().Model((*storage.Message)(nil)).
ColumnExpr("message.height, message.type, count(*)").
Where("message.height IN (?)", bun.In(heights)).
Where("message.time >= ?", startTime).
Where("message.time <= ?", endTime).
Group("message.type").
Group("message.height")

queryMsgsCounts = sortScope(queryMsgsCounts, "message.height", order)
if err = queryMsgsCounts.Scan(ctx, &listTypeCounts); err != nil {
return
}

for _, stat := range listTypeCounts {
blocksHeightMap[stat.Height].Stats.MessagesCounts[stat.Type] = stat.Count
}

err = query.Scan(ctx, &blocks)
return
}

Expand Down
21 changes: 0 additions & 21 deletions internal/storage/postgres/block_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ package postgres
import (
"context"

storageTypes "github.com/celenium-io/celestia-indexer/internal/storage/types"
pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types"

"github.com/celenium-io/celestia-indexer/internal/storage"
Expand All @@ -32,26 +31,6 @@ func (b *BlockStats) ByHeight(ctx context.Context, height pkgTypes.Level) (stats
Limit(1).
Scan(ctx)

if err != nil {
return
}

var msgsStats []typeCount
err = b.db.DB().NewSelect().Model((*storage.Message)(nil)).
ColumnExpr("message.type, count(*)").
Where("message.height = ?", height).
Group("message.type").
Scan(ctx, &msgsStats)

if err != nil {
return
}

stats.MessagesCounts = make(map[storageTypes.MsgType]int64)
for _, stat := range msgsStats {
stats.MessagesCounts[stat.Type] = stat.Count
}

return
}

Expand Down

0 comments on commit df3710b

Please sign in to comment.