Optimization: change data fields to bytes

cmd/api/routes_test.go

@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: 2024 PK Lab AG <contact@pklab.io>
+// SPDX-License-Identifier: MIT
+
 package main
 
 import (

go.mod

@@ -34,16 +34,16 @@ require (
 	github.com/labstack/echo/v4 v4.12.0
 	github.com/lib/pq v1.10.9
 	github.com/pkg/errors v0.9.1
-	github.com/rs/zerolog v1.31.0
+	github.com/rs/zerolog v1.32.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	github.com/swaggo/echo-swagger v1.4.0
 	github.com/swaggo/swag v1.16.1
 	github.com/tendermint/tendermint v0.34.29
-	github.com/uptrace/bun v1.1.17
-	github.com/uptrace/bun/dialect/pgdialect v1.1.17
-	github.com/uptrace/bun/driver/pgdriver v1.1.17
+	github.com/uptrace/bun v1.2.1
+	github.com/uptrace/bun/dialect/pgdialect v1.2.1
+	github.com/uptrace/bun/driver/pgdriver v1.2.1
 	go.opentelemetry.io/otel v1.24.0
 	go.opentelemetry.io/otel/sdk v1.24.0
 	go.opentelemetry.io/otel/trace v1.24.0
@@ -169,6 +169,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-getter v1.7.4 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
@@ -180,7 +181,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.17.3 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/klauspost/reedsolomon v1.12.1 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect

go.sum

@@ -721,6 +721,8 @@ github.com/hashicorp/go-getter v1.7.4/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
+github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
 github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
@@ -797,6 +799,8 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
 github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
 github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q=
@@ -971,8 +975,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
 github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
-github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
+github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -1074,12 +1078,12 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/uptrace/bun v1.1.17 h1:qxBaEIo0hC/8O3O6GrMDKxqyT+mw5/s0Pn/n6xjyGIk=
-github.com/uptrace/bun v1.1.17/go.mod h1:hATAzivtTIRsSJR4B8AXR+uABqnQxr3myKDKEf5iQ9U=
-github.com/uptrace/bun/dialect/pgdialect v1.1.17 h1:NsvFVHAx1Az6ytlAD/B6ty3cVE6j9Yp82bjqd9R9hOs=
-github.com/uptrace/bun/dialect/pgdialect v1.1.17/go.mod h1:fLBDclNc7nKsZLzNjFL6BqSdgJzbj2HdnyOnLoDvAME=
-github.com/uptrace/bun/driver/pgdriver v1.1.17 h1:hLj6WlvSZk5x45frTQnJrYtyhvgI6CA4r7gYdJ0gpn8=
-github.com/uptrace/bun/driver/pgdriver v1.1.17/go.mod h1:c9fa6FiiQjOe9mCaJC9NmFUE6vCGKTEsqrtLjPNz+kk=
+github.com/uptrace/bun v1.2.1 h1:2ENAcfeCfaY5+2e7z5pXrzFKy3vS8VXvkCag6N2Yzfk=
+github.com/uptrace/bun v1.2.1/go.mod h1:cNg+pWBUMmJ8rHnETgf65CEvn3aIKErrwOD6IA8e+Ec=
+github.com/uptrace/bun/dialect/pgdialect v1.2.1 h1:ceP99r03u+s8ylaDE/RzgcajwGiC76Jz3nS2ZgyPQ4M=
+github.com/uptrace/bun/dialect/pgdialect v1.2.1/go.mod h1:mv6B12cisvSc6bwKm9q9wcrr26awkZK8QXM+nso9n2U=
+github.com/uptrace/bun/driver/pgdriver v1.2.1 h1:Cp6c1tKzbTIyL8o0cGT6cOhTsmQZdsUNhgcV51dsmLU=
+github.com/uptrace/bun/driver/pgdriver v1.2.1/go.mod h1:jEd3WGx74hWLat3/IkesOoWNjrFNUDADK3nkyOFOOJM=
 github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk=
 github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
 github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=

internal/storage/event.go

@@ -8,15 +8,12 @@ import (
 	"time"
 
 	pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types"
-	jsoniter "github.com/json-iterator/go"
 
 	"github.com/celenium-io/celestia-indexer/internal/storage/types"
 	"github.com/dipdup-net/indexer-sdk/pkg/storage"
 	"github.com/uptrace/bun"
 )
 
-var json = jsoniter.ConfigCompatibleWithStandardLibrary
-
 type EventFilter struct {
 	Limit  int
 	Offset int
@@ -35,36 +32,16 @@ type IEvent interface {
 type Event struct {
 	bun.BaseModel `bun:"event" comment:"Table with celestia events."`
 
-	Id       uint64          `bun:"id,pk,notnull,autoincrement" comment:"Unique internal id"`
-	Height   pkgTypes.Level  `bun:"height,notnull"              comment:"The number (height) of this block" stats:"func:min max,filterable"`
-	Time     time.Time       `bun:"time,pk,notnull"             comment:"The time of block"                 stats:"func:min max,filterable"`
-	Position int64           `bun:"position"                    comment:"Position in transaction"`
-	Type     types.EventType `bun:",type:event_type"            comment:"Event type"                        stats:"filterable"`
-	TxId     *uint64         `bun:"tx_id"                       comment:"Transaction id"`
-	Data     map[string]any  `bun:"data,type:jsonb,nullzero"    comment:"Event data"`
+	Id       uint64          `bun:"id,pk,notnull,autoincrement"      comment:"Unique internal id"`
+	Height   pkgTypes.Level  `bun:"height,notnull"                   comment:"The number (height) of this block" stats:"func:min max,filterable"`
+	Time     time.Time       `bun:"time,pk,notnull"                  comment:"The time of block"                 stats:"func:min max,filterable"`
+	Position int64           `bun:"position"                         comment:"Position in transaction"`
+	Type     types.EventType `bun:",type:event_type"                 comment:"Event type"                        stats:"filterable"`
+	TxId     *uint64         `bun:"tx_id"                            comment:"Transaction id"`
+	Data     map[string]any  `bun:"data,msgpack,type:bytea,nullzero" comment:"Event data"`
 }
 
 // TableName -
 func (Event) TableName() string {
 	return "event"
 }
-
-func (e Event) Columns() []string {
-	return []string{
-		"height", "time", "position", "type",
-		"tx_id", "data",
-	}
-}
-
-func (e Event) Flat() []any {
-	data := []any{
-		e.Height, e.Time, e.Position, e.Type, e.TxId, nil,
-	}
-	if len(e.Data) > 0 {
-		raw, err := json.MarshalToString(e.Data)
-		if err == nil {
-			data[5] = raw
-		}
-	}
-	return data
-}

internal/storage/message.go

@@ -57,14 +57,14 @@ type IMessage interface {
 type Message struct {
 	bun.BaseModel `bun:"message" comment:"Table with celestia messages."`
 
-	Id       uint64         `bun:"id,pk,notnull,autoincrement" comment:"Unique internal id"`
-	Height   pkgTypes.Level `bun:",notnull"                    comment:"The number (height) of this block" stats:"func:min max,filterable"`
-	Time     time.Time      `bun:"time,pk,notnull"             comment:"The time of block"                 stats:"func:min max,filterable"`
-	Position int64          `bun:"position"                    comment:"Position in transaction"`
-	Type     types.MsgType  `bun:",type:msg_type"              comment:"Message type"                      stats:"filterable"`
-	TxId     uint64         `bun:"tx_id"                       comment:"Parent transaction id"`
-	Size     int            `bun:"size"                        comment:"Message size in bytes"`
-	Data     map[string]any `bun:"data,type:jsonb,nullzero"    comment:"Message data"`
+	Id       uint64            `bun:"id,pk,notnull,autoincrement" comment:"Unique internal id"`
+	Height   pkgTypes.Level    `bun:",notnull"                    comment:"The number (height) of this block" stats:"func:min max,filterable"`
+	Time     time.Time         `bun:"time,pk,notnull"             comment:"The time of block"                 stats:"func:min max,filterable"`
+	Position int64             `bun:"position"                    comment:"Position in transaction"`
+	Type     types.MsgType     `bun:",type:msg_type"              comment:"Message type"                      stats:"filterable"`
+	TxId     uint64            `bun:"tx_id"                       comment:"Parent transaction id"`
+	Size     int               `bun:"size"                        comment:"Message size in bytes"`
+	Data     types.PackedBytes `bun:"data,type:bytea,nullzero"    comment:"Message data"`
 
 	Namespace      []Namespace       `bun:"m2m:namespace_message,join:Message=Namespace"`
 	Addresses      []AddressWithType `bun:"-"`

internal/storage/postgres/index.go

@@ -330,15 +330,6 @@ func createIndices(ctx context.Context, conn *database.Bun) error {
 		}
 
 		// StakingLog
-		if _, err := tx.NewCreateIndex().
-			IfNotExists().
-			Model((*storage.StakingLog)(nil)).
-			Index("staking_log_address_id_idx").
-			Column("address_id").
-			Where("address_id is not null").
-			Exec(ctx); err != nil {
-			return err
-		}
 		if _, err := tx.NewCreateIndex().
 			IfNotExists().
 			Model((*storage.StakingLog)(nil)).

internal/storage/postgres/staking_log.go

@@ -4,8 +4,6 @@
 package postgres
 
 import (
-	"context"
-
 	"github.com/celenium-io/celestia-indexer/internal/storage"
 	"github.com/dipdup-net/go-lib/database"
 	"github.com/dipdup-net/indexer-sdk/pkg/storage/postgres"
@@ -22,17 +20,3 @@ func NewStakingLog(db *database.Bun) *StakingLog {
 		Table: postgres.NewTable[*storage.StakingLog](db),
 	}
 }
-
-func (d *StakingLog) ByValidator(ctx context.Context, id uint64, limit, offset int) (logs []storage.StakingLog, err error) {
-	query := d.DB().NewSelect().
-		Model(&logs).
-		Where("validator_id = ?", id)
-
-	query = limitScope(query, limit)
-	if offset > 0 {
-		query = query.Offset(offset)
-	}
-
-	err = query.Scan(ctx)
-	return
-}

internal/storage/postgres/transaction.go

@@ -8,17 +8,15 @@ import (
 	"time"
 
 	"github.com/celenium-io/celestia-indexer/pkg/types"
-	jsoniter "github.com/json-iterator/go"
 	"github.com/lib/pq"
 	"github.com/shopspring/decimal"
 	"github.com/uptrace/bun"
+	"github.com/vmihailenco/msgpack/v5"
 
 	models "github.com/celenium-io/celestia-indexer/internal/storage"
 	"github.com/dipdup-net/indexer-sdk/pkg/storage"
 )
 
-var json = jsoniter.ConfigCompatibleWithStandardLibrary
-
 type Transaction struct {
 	storage.Transaction
 }
@@ -161,11 +159,10 @@ func (tx Transaction) SaveEvents(ctx context.Context, events ...models.Event) er
 		}
 
 		for i := range events {
-			var s *string
+			var s []byte
 			if len(events[i].Data) > 0 {
-				raw, err := json.MarshalToString(events[i].Data)
-				if err == nil {
-					s = &raw
+				if raw, err := msgpack.Marshal(events[i].Data); err == nil {
+					s = raw
 				}
 			}
 

internal/storage/staking_log.go

@@ -4,7 +4,6 @@
 package storage
 
 import (
-	"context"
 	"time"
 
 	"github.com/celenium-io/celestia-indexer/internal/storage/types"
@@ -17,8 +16,6 @@ import (
 //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed
 type IStakingLog interface {
 	storage.Table[*StakingLog]
-
-	ByValidator(ctx context.Context, validatorId uint64, limit, offset int) ([]StakingLog, error)
 }
 
 // Delegation -

internal/storage/types/packed_bytes.go

@@ -0,0 +1,34 @@
+package types
+
+import (
+	"database/sql"
+	"database/sql/driver"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
+)
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+type PackedBytes map[string]any
+
+var _ sql.Scanner = (*PackedBytes)(nil)
+
+func (pb *PackedBytes) Scan(src interface{}) error {
+	b, ok := src.([]byte)
+	if !ok {
+		return errors.Errorf("invalid packed bytes type: %T", src)
+	}
+
+	return json.Unmarshal(b, pb)
+}
+
+var _ driver.Valuer = (*PackedBytes)(nil)
+
+func (pb PackedBytes) Value() (driver.Value, error) {
+	return pb.ToBytes()
+}
+
+func (pb PackedBytes) ToBytes() ([]byte, error) {
+	return json.Marshal(pb)
+}