From 2581772afb6875de2a6f4d54a93a2f11be5ab2fc Mon Sep 17 00:00:00 2001 From: K-KAD <54041472+k-kaddal@users.noreply.github.com> Date: Thu, 4 Apr 2024 08:13:15 +0100 Subject: [PATCH] feat(blobstorage): allow get_blob api to return blob data (#16629) Co-authored-by: jeff <113397187+cyberhorsey@users.noreply.github.com> Co-authored-by: Jeffery Walsh --- packages/blobstorage/.default.indexer.env | 4 ++-- packages/blobstorage/blob_hash.go | 2 -- packages/blobstorage/indexer/beaconclient.go | 4 ++++ packages/blobstorage/indexer/indexer.go | 24 +++++++++---------- .../16667_create_blob_hashes_table.sql | 1 - packages/blobstorage/pkg/http/get_blob.go | 13 ++++++---- packages/blobstorage/pkg/repo/blob_hash.go | 1 - 7 files changed, 26 insertions(+), 23 deletions(-) diff --git a/packages/blobstorage/.default.indexer.env b/packages/blobstorage/.default.indexer.env index c228b59292..a6cf895557 100644 --- a/packages/blobstorage/.default.indexer.env +++ b/packages/blobstorage/.default.indexer.env @@ -1,5 +1,5 @@ -RPC_URL=wss://l1ws.internal.taiko.xyz -BEACON_URL=https://l1beacon.internal.taiko.xyz +RPC_URL=wss://l1ws.hekla.taiko.xyz +BEACON_URL=https://l1beacon.hekla.taiko.xyz TAIKO_L1_CONTRACT_ADDRESS=0xC069c3d2a9f2479F559AD34485698ad5199C555f DATABASE_HOST=localhost DATABASE_PORT=3306 diff --git a/packages/blobstorage/blob_hash.go b/packages/blobstorage/blob_hash.go index d8e7ab4408..5b4d9a4bd3 100644 --- a/packages/blobstorage/blob_hash.go +++ b/packages/blobstorage/blob_hash.go @@ -3,7 +3,6 @@ package blobstorage type BlobHash struct { BlobHash string KzgCommitment string - BlockTimestamp uint64 BlobData string BlockID uint64 EmittedBlockID uint64 @@ -12,7 +11,6 @@ type BlobHash struct { type SaveBlobHashOpts struct { BlobHash string KzgCommitment string - BlockTimestamp uint64 BlobData string BlockID uint64 EmittedBlockID uint64 diff --git a/packages/blobstorage/indexer/beaconclient.go b/packages/blobstorage/indexer/beaconclient.go index ea2548aeeb..8e5ca075bc 100644 --- a/packages/blobstorage/indexer/beaconclient.go +++ b/packages/blobstorage/indexer/beaconclient.go @@ -8,6 +8,8 @@ import ( "net/http" "strconv" "time" + + "golang.org/x/exp/slog" ) var ( @@ -64,6 +66,8 @@ func NewBeaconClient(cfg *Config, timeout time.Duration) (*BeaconClient, error) return nil, err } + slog.Info("beaconClientInfo", "secondsPerSlot", secondsPerSlotUint64, "genesisTime", genesisTime) + return &BeaconClient{ beaconURL: cfg.BeaconURL, genesisTime: genesisTime, diff --git a/packages/blobstorage/indexer/indexer.go b/packages/blobstorage/indexer/indexer.go index 6d5752bce6..cee4fb18e1 100644 --- a/packages/blobstorage/indexer/indexer.go +++ b/packages/blobstorage/indexer/indexer.go @@ -284,18 +284,23 @@ func (i *Indexer) checkReorg(ctx context.Context, event *taikol1.TaikoL1BlockPro } func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProposed) error { - blockID, err := i.beaconClient.timeToSlot(event.Meta.Timestamp) + slot, err := i.beaconClient.timeToSlot(event.Meta.Timestamp) if err != nil { return err } - slog.Info("blockProposed event found", "blockID", blockID, "emittedIn", event.Raw.BlockNumber, "blobUsed", event.Meta.BlobUsed) + slog.Info("blockProposed event found", + "slot", slot, + "emittedIn", event.Raw.BlockNumber, + "blobUsed", event.Meta.BlobUsed, + "timesStamp", event.Meta.Timestamp, + ) if !event.Meta.BlobUsed { return nil } - blobsResponse, err := i.beaconClient.getBlobs(ctx, blockID) + blobsResponse, err := i.beaconClient.getBlobs(ctx, slot) if err != nil { return err } @@ -306,15 +311,9 @@ func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProp metaBlobHash := common.BytesToHash(event.Meta.BlobHash[:]) // Comparing the hex strings of meta.blobHash (blobHash) if calculateBlobHash(data.KzgCommitment) == metaBlobHash { - blockTs, err := i.getBlockTimestamp(i.cfg.RPCURL, new(big.Int).SetUint64(blockID)) - if err != nil { - slog.Error("error getting block timestamp", "error", err) - return err - } - - slog.Info("blockHash", "blobHash", metaBlobHash.String()) + slog.Info("storing blobHash in db", "blobHash", metaBlobHash.String()) - err = i.storeBlobInDB(metaBlobHash.String(), data.KzgCommitment, data.Blob, blockTs, event.BlockId.Uint64(), event.Raw.BlockNumber) + err = i.storeBlobInDB(metaBlobHash.String(), data.KzgCommitment, data.Blob, event.BlockId.Uint64(), event.Raw.BlockNumber) if err != nil { slog.Error("Error storing blob in DB", "error", err) return err @@ -327,13 +326,12 @@ func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProp return errors.New("BLOB not found") } -func (i *Indexer) storeBlobInDB(blobHashInMeta, kzgCommitment, blob string, blockTs uint64, blockID uint64, emittedBlockID uint64) error { +func (i *Indexer) storeBlobInDB(blobHashInMeta, kzgCommitment, blob string, blockID uint64, emittedBlockID uint64) error { return i.blobHashRepo.Save(blobstorage.SaveBlobHashOpts{ BlobHash: blobHashInMeta, KzgCommitment: kzgCommitment, BlockID: blockID, BlobData: blob, - BlockTimestamp: blockTs, EmittedBlockID: emittedBlockID, }) } diff --git a/packages/blobstorage/migrations/16667_create_blob_hashes_table.sql b/packages/blobstorage/migrations/16667_create_blob_hashes_table.sql index a2b657017a..be1b4b63c9 100644 --- a/packages/blobstorage/migrations/16667_create_blob_hashes_table.sql +++ b/packages/blobstorage/migrations/16667_create_blob_hashes_table.sql @@ -6,7 +6,6 @@ CREATE TABLE IF NOT EXISTS blob_hashes ( emitted_block_id BIGINT NOT NULL, blob_hash VARCHAR(100) NOT NULL, kzg_commitment LONGTEXT NOT NULL, - block_timestamp BIGINT NOT NULL, blob_data LONGTEXT NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP diff --git a/packages/blobstorage/pkg/http/get_blob.go b/packages/blobstorage/pkg/http/get_blob.go index c01a00e7a0..89d0266355 100644 --- a/packages/blobstorage/pkg/http/get_blob.go +++ b/packages/blobstorage/pkg/http/get_blob.go @@ -15,8 +15,9 @@ type getBlobResponse struct { } type blobData struct { - Blob string `bson:"blob_hash" json:"blob_hash"` + BlobHash string `bson:"blob_hash" json:"blob_hash"` KzgCommitment string `bson:"kzg_commitment" json:"kzg_commitment"` + Blob string `bson:"blob" json:"blob"` } // GetBlob @@ -37,6 +38,7 @@ func (srv *Server) GetBlob(c echo.Context) error { } data, err := srv.getBlobData(strings.Split(blobHashes, ",")) + if err != nil { return webutils.LogAndRenderErrors(c, http.StatusBadRequest, err) } @@ -48,8 +50,9 @@ func (srv *Server) GetBlob(c echo.Context) error { // Convert data to the correct type for _, d := range data { response.Data = append(response.Data, blobData{ - Blob: d.Blob, + BlobHash: d.BlobHash, KzgCommitment: d.KzgCommitment, + Blob: d.Blob, }, ) } @@ -69,15 +72,17 @@ func (srv *Server) getBlobData(blobHashes []string) ([]blobData, error) { if err != nil { if err == gorm.ErrRecordNotFound { // Handle case where blob hash is not found - result.Blob = "NOT_FOUND" + result.BlobHash = "NOT_FOUND" result.KzgCommitment = "NOT_FOUND" + result.Blob = "NOT_FOUND" } else { // Return error for other types of errors return nil, err } } else { - result.Blob = bh.BlobHash + result.BlobHash = bh.BlobHash result.KzgCommitment = bh.KzgCommitment + result.Blob = bh.BlobData results = append(results, result) } diff --git a/packages/blobstorage/pkg/repo/blob_hash.go b/packages/blobstorage/pkg/repo/blob_hash.go index eaeb04010d..273bd906b9 100644 --- a/packages/blobstorage/pkg/repo/blob_hash.go +++ b/packages/blobstorage/pkg/repo/blob_hash.go @@ -27,7 +27,6 @@ func (r *BlobHashRepository) Save(opts blobstorage.SaveBlobHashOpts) error { b := &blobstorage.BlobHash{ BlobHash: opts.BlobHash, KzgCommitment: opts.KzgCommitment, - BlockTimestamp: opts.BlockTimestamp, BlobData: opts.BlobData, BlockID: opts.BlockID, EmittedBlockID: opts.EmittedBlockID,