From 7143c1de590998623a1d3243b25d672f51675b00 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 15 Oct 2025 16:32:41 -0500 Subject: [PATCH 1/5] chore: update go-eth2-client --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 257e111..9343bd3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/base/blob-archiver go 1.24.0 require ( - github.com/attestantio/go-eth2-client v0.24.0 + github.com/attestantio/go-eth2-client v0.27.1 github.com/ethereum-optimism/optimism v1.11.2 github.com/ethereum/go-ethereum v1.101500.1 github.com/go-chi/chi/v5 v5.0.12 @@ -80,13 +80,13 @@ require ( go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect - golang.org/x/crypto v0.32.0 // indirect + golang.org/x/crypto v0.33.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/net v0.34.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/term v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/Knetic/govaluate.v3 v3.0.0 // indirect diff --git a/go.sum b/go.sum index cdf0fc0..b622399 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= -github.com/attestantio/go-eth2-client v0.24.0 h1:lGVbcnhlBwRglt1Zs56JOCgXVyLWKFZOmZN8jKhE7Ws= -github.com/attestantio/go-eth2-client v0.24.0/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/attestantio/go-eth2-client v0.27.1 h1:g7bm+gG/p+gfzYdEuxuAepVWYb8EO+2KojV5/Lo2BxM= +github.com/attestantio/go-eth2-client v0.27.1/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= @@ -267,8 +267,8 @@ go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLk golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -276,8 +276,8 @@ golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -293,17 +293,17 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From a65b4753825362953857ce2e7260368b9be91720 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 15 Oct 2025 21:52:45 -0500 Subject: [PATCH 2/5] feat: migrate to beacon blobs endpoint with valid KZG test data Implements support for the new /eth/v1/beacon/blobs/{block_id} endpoint with automatic fallback to the legacy blob_sidecars endpoint. Key changes: - Archiver: Prefer blobs endpoint, compute KZG commitments/proofs from blob data, fall back to blob_sidecars on failure - API: Add /eth/v1/beacon/blobs/{id} endpoint with versioned_hashes filtering support using kzg4844.CalcBlobHashV1 - Test data: Generate valid BLS12-381 field elements by reducing random data modulo the field modulus to ensure KZG operations succeed - Test helpers: Consolidate blob generation into NewBlobSidecars with proper header support for consistent test comparisons All tests now use the blobs endpoint (except one that explicitly tests the fallback behavior). The archiver stores complete blob sidecars with derived KZG commitments and proofs, maintaining backward compatibility with existing storage format. Technical notes: - Each 32-byte chunk in blob data must be < BLS12-381 field modulus - KZG commitments/proofs are computed using gokzg4844 library - Versioned hashes use ethereum/go-ethereum's kzg4844.CalcBlobHashV1 - Test data includes valid headers to match reconstructed sidecars --- api/service/api.go | 113 +++++++++++++++++++++++++ api/service/api_test.go | 135 +++++++++++++++++++++++++++++- archiver/service/archiver.go | 82 ++++++++++++++++-- archiver/service/archiver_test.go | 99 ++++++++++++++++------ common/beacon/beacontest/stub.go | 67 +++++++++++---- common/beacon/client.go | 1 + common/blobtest/helpers.go | 77 ++++++++++++++--- go.mod | 2 + go.sum | 4 +- validator/service/service_test.go | 6 +- 10 files changed, 515 insertions(+), 71 deletions(-) diff --git a/api/service/api.go b/api/service/api.go index cc597e9..35fc458 100644 --- a/api/service/api.go +++ b/api/service/api.go @@ -2,6 +2,7 @@ package service import ( "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -13,6 +14,7 @@ import ( client "github.com/attestantio/go-eth2-client" "github.com/attestantio/go-eth2-client/api" + v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec/deneb" m "github.com/base/blob-archiver/api/metrics" "github.com/base/blob-archiver/api/version" @@ -20,6 +22,7 @@ import ( opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -107,6 +110,7 @@ func NewAPI(dataStoreClient storage.DataStoreReader, beaconClient client.BeaconB }) r.Get("/eth/v1/beacon/blob_sidecars/{id}", result.blobSidecarHandler) + r.Get("/eth/v1/beacon/blobs/{id}", result.blobsHandler) r.Get("/eth/v1/node/version", result.versionHandler) return result @@ -264,3 +268,112 @@ func filterBlobs(blobs []*deneb.BlobSidecar, _indices []string) ([]*deneb.BlobSi return filteredBlobs, nil } + +// filterBlobsByVersionedHashes filters sidecars by versioned hashes query parameter. +// Returns the filtered sidecars in the order they were requested, or all sidecars if no hashes provided. +func filterBlobsByVersionedHashes(sidecars []*deneb.BlobSidecar, _versionedHashes []string) ([]*deneb.BlobSidecar, *httpError) { + var versionedHashes []string + if len(_versionedHashes) == 0 { + return sidecars, nil + } else if len(_versionedHashes) == 1 { + versionedHashes = strings.Split(_versionedHashes[0], ",") + } else { + versionedHashes = _versionedHashes + } + + // Build map of commitment hash -> sidecar for quick lookup + // CalcBlobHashV1 requires a sha256 hasher instance + hasher := sha256.New() + hashToSidecar := make(map[[32]byte]*deneb.BlobSidecar) + for _, sidecar := range sidecars { + hasher.Reset() + commitment := kzg4844.Commitment(sidecar.KZGCommitment) + vh := kzg4844.CalcBlobHashV1(hasher, &commitment) + hashToSidecar[vh] = sidecar + } + + // Return sidecars in the order of requested hashes + filteredBlobs := make([]*deneb.BlobSidecar, 0, len(versionedHashes)) + for _, hashStr := range versionedHashes { + hash := common.HexToHash(hashStr) + var versionedHash [32]byte + copy(versionedHash[:], hash[:]) + + if sidecar, ok := hashToSidecar[versionedHash]; ok { + filteredBlobs = append(filteredBlobs, sidecar) + } + } + + return filteredBlobs, nil +} + +// sidecarsToBlobs converts blob sidecars to a Blobs response by extracting only the blob data +func sidecarsToBlobs(sidecars []*deneb.BlobSidecar) v1.Blobs { + blobs := make(v1.Blobs, len(sidecars)) + for i, sidecar := range sidecars { + blobs[i] = &sidecar.Blob + } + return blobs +} + +// blobsHandler implements the /eth/v1/beacon/blobs/{id} endpoint, using the underlying DataStoreReader +// to fetch blobs instead of the beacon node. This endpoint serves blobs without KZG proofs. +// Filtering by versioned_hashes query parameter is supported (per EIP-4844). +func (a *API) blobsHandler(w http.ResponseWriter, r *http.Request) { + param := chi.URLParam(r, "id") + beaconBlockHash, err := a.toBeaconBlockHash(param) + if err != nil { + err.write(w) + return + } + + result, storageErr := a.dataStoreClient.ReadBlob(r.Context(), beaconBlockHash) + if storageErr != nil { + if errors.Is(storageErr, storage.ErrNotFound) { + errUnknownBlock.write(w) + } else { + a.logger.Info("unexpected error fetching blobs", "err", storageErr, "beaconBlockHash", beaconBlockHash.String(), "param", param) + errServerError.write(w) + } + return + } + + sidecars := result.BlobSidecars.Data + + // Filter by versioned_hashes query parameter (not indices) + filteredSidecars, err := filterBlobsByVersionedHashes(sidecars, r.URL.Query()["versioned_hashes"]) + if err != nil { + err.write(w) + return + } + + // Convert sidecars to blobs + blobs := sidecarsToBlobs(filteredSidecars) + responseType := r.Header.Get("Accept") + + if responseType == sszAcceptType { + w.Header().Set("Content-Type", sszAcceptType) + res, err := blobs.MarshalSSZ() + if err != nil { + a.logger.Error("unable to marshal blobs to SSZ", "err", err) + errServerError.write(w) + return + } + + _, err = w.Write(res) + + if err != nil { + a.logger.Error("unable to write ssz response", "err", err) + errServerError.write(w) + return + } + } else { + w.Header().Set("Content-Type", jsonAcceptType) + err := json.NewEncoder(w).Encode(blobs) + if err != nil { + a.logger.Error("unable to encode blobs to JSON", "err", err) + errServerError.write(w) + return + } + } +} diff --git a/api/service/api_test.go b/api/service/api_test.go index 8a789ad..b6a128e 100644 --- a/api/service/api_test.go +++ b/api/service/api_test.go @@ -3,6 +3,7 @@ package service import ( "compress/gzip" "context" + "crypto/sha256" "encoding/json" "fmt" "io" @@ -21,6 +22,7 @@ import ( "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/testlog" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" "github.com/stretchr/testify/require" ) @@ -80,7 +82,7 @@ func TestAPIService(t *testing.T) { BeaconBlockHash: rootOne, }, BlobSidecars: storage.BlobSidecars{ - Data: blobtest.NewBlobSidecars(t, 2), + Data: blobtest.NewBlobSidecars(t, 2, nil), }, } @@ -89,7 +91,7 @@ func TestAPIService(t *testing.T) { BeaconBlockHash: rootTwo, }, BlobSidecars: storage.BlobSidecars{ - Data: blobtest.NewBlobSidecars(t, 2), + Data: blobtest.NewBlobSidecars(t, 2, nil), }, } @@ -99,7 +101,7 @@ func TestAPIService(t *testing.T) { BeaconBlockHash: rootThree, }, BlobSidecars: storage.BlobSidecars{ - Data: blobtest.NewBlobSidecars(t, 8), // More than 6 blobs + Data: blobtest.NewBlobSidecars(t, 8, nil), // More than 6 blobs }, } @@ -364,3 +366,130 @@ func TestHealthHandler(t *testing.T) { require.Equal(t, 200, response.Code) } + +func TestBlobsHandlerJSON(t *testing.T) { + a, fs, _, cleanup := setup(t) + defer cleanup() + + // Pre-populate storage with blob sidecars + testBlobs := blobtest.NewBlobSidecars(t, 3, nil) + data := storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: blobtest.Five, + }, + BlobSidecars: storage.BlobSidecars{ + Data: testBlobs, + }, + } + err := fs.WriteBlob(context.Background(), data) + require.NoError(t, err) + + // Request blobs endpoint with JSON encoding + request := httptest.NewRequest("GET", "/eth/v1/beacon/blobs/"+blobtest.Five.String(), nil) + request.Header.Set("Accept", "application/json") + response := httptest.NewRecorder() + + a.router.ServeHTTP(response, request) + + require.Equal(t, 200, response.Code) + require.Equal(t, "application/json", response.Header().Get("Content-Type")) + + var blobs v1.Blobs + err = json.Unmarshal(response.Body.Bytes(), &blobs) + require.NoError(t, err) + require.Equal(t, len(testBlobs), len(blobs)) + + // Verify blob data matches + for i, blob := range blobs { + require.Equal(t, testBlobs[i].Blob, *blob) + } +} + +func TestBlobsHandlerSSZ(t *testing.T) { + a, fs, _, cleanup := setup(t) + defer cleanup() + + // Pre-populate storage with blob sidecars + testBlobs := blobtest.NewBlobSidecars(t, 2, nil) + data := storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: blobtest.One, + }, + BlobSidecars: storage.BlobSidecars{ + Data: testBlobs, + }, + } + err := fs.WriteBlob(context.Background(), data) + require.NoError(t, err) + + // Request blobs endpoint with SSZ encoding + request := httptest.NewRequest("GET", "/eth/v1/beacon/blobs/"+blobtest.One.String(), nil) + request.Header.Set("Accept", "application/octet-stream") + response := httptest.NewRecorder() + + a.router.ServeHTTP(response, request) + + require.Equal(t, 200, response.Code) + require.Equal(t, "application/octet-stream", response.Header().Get("Content-Type")) + + // Unmarshal SSZ response + blobs := v1.Blobs{} + err = blobs.UnmarshalSSZ(response.Body.Bytes()) + require.NoError(t, err) + require.Equal(t, len(testBlobs), len(blobs)) +} + +func TestBlobsHandlerWithVersionedHashes(t *testing.T) { + a, fs, _, cleanup := setup(t) + defer cleanup() + + // Pre-populate storage with blob sidecars + testBlobs := blobtest.NewBlobSidecars(t, 5, nil) + data := storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: blobtest.Four, + }, + BlobSidecars: storage.BlobSidecars{ + Data: testBlobs, + }, + } + err := fs.WriteBlob(context.Background(), data) + require.NoError(t, err) + + // Compute versioned hashes from commitments + hasher := sha256.New() + commitment0 := kzg4844.Commitment(testBlobs[0].KZGCommitment) + vh0 := kzg4844.CalcBlobHashV1(hasher, &commitment0) + + hasher.Reset() + commitment2 := kzg4844.Commitment(testBlobs[2].KZGCommitment) + vh2 := kzg4844.CalcBlobHashV1(hasher, &commitment2) + + // Request blobs endpoint with versioned_hashes filter + request := httptest.NewRequest("GET", fmt.Sprintf("/eth/v1/beacon/blobs/%s?versioned_hashes=%s&versioned_hashes=%s", blobtest.Four.String(), common.Hash(vh0).Hex(), common.Hash(vh2).Hex()), nil) + request.Header.Set("Accept", "application/json") + response := httptest.NewRecorder() + + a.router.ServeHTTP(response, request) + + require.Equal(t, 200, response.Code) + + var blobs v1.Blobs + err = json.Unmarshal(response.Body.Bytes(), &blobs) + require.NoError(t, err) + require.Equal(t, 2, len(blobs)) +} + +func TestBlobsHandlerNotFound(t *testing.T) { + a, _, _, cleanup := setup(t) + defer cleanup() + + // Request blobs endpoint for non-existent block + request := httptest.NewRequest("GET", "/eth/v1/beacon/blobs/"+blobtest.Seven.String(), nil) + request.Header.Set("Accept", "application/json") + response := httptest.NewRecorder() + + a.router.ServeHTTP(response, request) + + require.Equal(t, 404, response.Code) +} diff --git a/archiver/service/archiver.go b/archiver/service/archiver.go index 704ee44..487d79d 100644 --- a/archiver/service/archiver.go +++ b/archiver/service/archiver.go @@ -9,9 +9,11 @@ import ( client "github.com/attestantio/go-eth2-client" "github.com/attestantio/go-eth2-client/api" v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/base/blob-archiver/archiver/flags" "github.com/base/blob-archiver/archiver/metrics" "github.com/base/blob-archiver/common/storage" + gokzg4844 "github.com/crate-crypto/go-kzg-4844" "github.com/ethereum-optimism/optimism/op-service/retry" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -25,9 +27,47 @@ const ( backfillErrorRetryInterval = 5 * time.Second ) +// blobsToSidecars converts blobs to blob sidecars by computing KZG commitments and proofs from the blob data. +// The blobs and commitments are ordered identically (by KZG commitment order in the block). +// The header information is included from the provided header. +func blobsToSidecars(blobs v1.Blobs, header *v1.BeaconBlockHeader) ([]*deneb.BlobSidecar, error) { + kzgCtx, err := gokzg4844.NewContext4096Secure() + if err != nil { + return nil, err + } + + sidecars := make([]*deneb.BlobSidecar, len(blobs)) + for i, blob := range blobs { + // Cast to gokzg4844.Blob for KZG operations + kzgBlob := (*gokzg4844.Blob)(blob) + + // Compute KZG commitment from blob data + commitment, err := kzgCtx.BlobToKZGCommitment(kzgBlob, 0) + if err != nil { + return nil, err + } + + // Compute KZG proof from blob data and commitment + proof, err := kzgCtx.ComputeBlobKZGProof(kzgBlob, commitment, 0) + if err != nil { + return nil, err + } + + sidecars[i] = &deneb.BlobSidecar{ + Index: deneb.BlobIndex(i), + Blob: *blob, + KZGCommitment: deneb.KZGCommitment(commitment), + KZGProof: deneb.KZGProof(proof), + SignedBlockHeader: header.Header, + } + } + return sidecars, nil +} + type BeaconClient interface { client.BlobSidecarsProvider client.BeaconBlockHeadersProvider + client.BlobsProvider } func NewArchiver(l log.Logger, cfg flags.ArchiverConfig, dataStoreClient storage.DataStore, client BeaconClient, m metrics.Metricer) (*Archiver, error) { @@ -84,6 +124,9 @@ func (a *Archiver) Stop(ctx context.Context) error { // If the blobs are already stored, it will not overwrite the data. Currently, the archiver does not // perform any validation of the blobs, it assumes a trusted beacon node. See: // https://github.com/base/blob-archiver/issues/4. +// +// The function prefers the /eth/v1/beacon/blobs endpoint but falls back to /eth/v1/beacon/blob_sidecars +// if the blobs endpoint fails. func (a *Archiver) persistBlobsForBlockToS3(ctx context.Context, blockIdentifier string, overwrite bool) (*v1.BeaconBlockHeader, bool, error) { currentHeader, err := a.beaconClient.BeaconBlockHeader(ctx, &api.BeaconBlockHeaderOpts{ Block: blockIdentifier, @@ -105,22 +148,47 @@ func (a *Archiver) persistBlobsForBlockToS3(ctx context.Context, blockIdentifier return currentHeader.Data, true, nil } - blobSidecars, err := a.beaconClient.BlobSidecars(ctx, &api.BlobSidecarsOpts{ + // Try the new blobs endpoint first + var blobSidecarData []*deneb.BlobSidecar + blobs, err := a.beaconClient.Blobs(ctx, &api.BlobsOpts{ Block: currentHeader.Data.Root.String(), }) - if err != nil { - a.log.Error("failed to fetch blob sidecars", "err", err) - return nil, false, err + if err == nil && blobs != nil && blobs.Data != nil && len(blobs.Data) > 0 { + // Successfully fetched blobs, compute commitments and proofs from blob data + a.log.Debug("fetched blobs from blobs endpoint, computing KZG commitments and proofs", "count", len(blobs.Data)) + + var err error + blobSidecarData, err = blobsToSidecars(blobs.Data, currentHeader.Data) + if err != nil { + a.log.Error("failed to compute KZG commitments and proofs for blobs", "err", err) + return nil, false, err + } + } else { + // Fall back to blob sidecars endpoint + if err != nil { + a.log.Debug("blobs endpoint failed, falling back to blob sidecars", "err", err) + } + + blobSidecarsResp, fallbackErr := a.beaconClient.BlobSidecars(ctx, &api.BlobSidecarsOpts{ + Block: currentHeader.Data.Root.String(), + }) + + if fallbackErr != nil { + a.log.Error("failed to fetch blob sidecars", "err", fallbackErr) + return nil, false, fallbackErr + } + + blobSidecarData = blobSidecarsResp.Data } - a.log.Debug("fetched blob sidecars", "count", len(blobSidecars.Data)) + a.log.Debug("fetched blob sidecars", "count", len(blobSidecarData)) blobData := storage.BlobData{ Header: storage.Header{ BeaconBlockHash: common.Hash(currentHeader.Data.Root), }, - BlobSidecars: storage.BlobSidecars{Data: blobSidecars.Data}, + BlobSidecars: storage.BlobSidecars{Data: blobSidecarData}, } // The blob that is being written has not been validated. It is assumed that the beacon node is trusted. @@ -131,7 +199,7 @@ func (a *Archiver) persistBlobsForBlockToS3(ctx context.Context, blockIdentifier return nil, false, err } - a.metrics.RecordStoredBlobs(len(blobSidecars.Data)) + a.metrics.RecordStoredBlobs(len(blobSidecarData)) return currentHeader.Data, exists, nil } diff --git a/archiver/service/archiver_test.go b/archiver/service/archiver_test.go index 5453b1d..680a16f 100644 --- a/archiver/service/archiver_test.go +++ b/archiver/service/archiver_test.go @@ -31,7 +31,8 @@ func setup(t *testing.T, beacon *beacontest.StubBeaconClient) (*Archiver, *stora } func TestArchiver_FetchAndPersist(t *testing.T) { - svc, fs := setup(t, beacontest.NewDefaultStubBeaconClient(t)) + beacon := beacontest.NewDefaultStubBeaconClient(t) + svc, fs := setup(t, beacon) fs.CheckNotExistsOrFail(t, blobtest.OriginBlock) @@ -62,21 +63,21 @@ func TestArchiver_FetchAndPersistOverwriting(t *testing.T) { BeaconBlockHash: blobtest.Five, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Five.String()], + Data: beacon.SidecarsByBlock[blobtest.Five.String()], }, }) - require.Equal(t, fs.ReadOrFail(t, blobtest.Five).BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()]) + require.Equal(t, fs.ReadOrFail(t, blobtest.Five).BlobSidecars.Data, beacon.SidecarsByBlock[blobtest.Five.String()]) // change the blob data -- this isn't possible w/out changing the hash. But it allows us to test the overwrite - beacon.Blobs[blobtest.Five.String()] = blobtest.NewBlobSidecars(t, 6) + beacon.SidecarsByBlock[blobtest.Five.String()] = blobtest.NewBlobSidecars(t, 6, beacon.Headers[blobtest.Five.String()].Header) _, exists, err := svc.persistBlobsForBlockToS3(context.Background(), blobtest.Five.String(), true) require.NoError(t, err) require.True(t, exists) // It should have overwritten the blob data - require.Equal(t, fs.ReadOrFail(t, blobtest.Five).BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()]) + require.Equal(t, fs.ReadOrFail(t, blobtest.Five).BlobSidecars.Data, beacon.SidecarsByBlock[blobtest.Five.String()]) // Overwriting a non-existent blob should return exists=false _, exists, err = svc.persistBlobsForBlockToS3(context.Background(), blobtest.Four.String(), true) @@ -94,7 +95,7 @@ func TestArchiver_BackfillToOrigin(t *testing.T) { BeaconBlockHash: blobtest.Five, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Five.String()], + Data: beacon.SidecarsByBlock[blobtest.Five.String()], }, }) require.NoError(t, err) @@ -110,7 +111,7 @@ func TestArchiver_BackfillToOrigin(t *testing.T) { for _, blob := range expectedBlobs { fs.CheckExistsOrFail(t, blob) data := fs.ReadOrFail(t, blob) - require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[blob.String()]) + require.Equal(t, data.BlobSidecars.Data, beacon.SidecarsByBlock[blob.String()]) } } @@ -124,7 +125,7 @@ func TestArchiver_BackfillToExistingBlock(t *testing.T) { BeaconBlockHash: blobtest.Five, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Five.String()], + Data: beacon.SidecarsByBlock[blobtest.Five.String()], }, }) require.NoError(t, err) @@ -135,7 +136,7 @@ func TestArchiver_BackfillToExistingBlock(t *testing.T) { BeaconBlockHash: blobtest.One, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.One.String()], + Data: beacon.SidecarsByBlock[blobtest.One.String()], }, }) require.NoError(t, err) @@ -159,7 +160,7 @@ func TestArchiver_BackfillToExistingBlock(t *testing.T) { data, err := fs.ReadBlob(context.Background(), blob) require.NoError(t, err) require.NotNil(t, data) - require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[blob.String()]) + require.Equal(t, data.BlobSidecars.Data, beacon.SidecarsByBlock[blob.String()]) } } @@ -191,7 +192,7 @@ func TestArchiver_BackfillFinishOldProcess(t *testing.T) { BeaconBlockHash: blobtest.Five, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Five.String()], + Data: beacon.SidecarsByBlock[blobtest.Five.String()], }, }) require.NoError(t, err) @@ -202,7 +203,7 @@ func TestArchiver_BackfillFinishOldProcess(t *testing.T) { BeaconBlockHash: blobtest.Three, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Three.String()], + Data: beacon.SidecarsByBlock[blobtest.Three.String()], }, }) require.NoError(t, err) @@ -213,7 +214,7 @@ func TestArchiver_BackfillFinishOldProcess(t *testing.T) { BeaconBlockHash: blobtest.One, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.One.String()], + Data: beacon.SidecarsByBlock[blobtest.One.String()], }, }) require.NoError(t, err) @@ -250,7 +251,7 @@ func TestArchiver_BackfillFinishOldProcess(t *testing.T) { data, err := fs.ReadBlob(context.Background(), blob) require.NoError(t, err) require.NotNil(t, data) - require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[blob.String()]) + require.Equal(t, data.BlobSidecars.Data, beacon.SidecarsByBlock[blob.String()]) } actualProcesses, err = svc.dataStoreClient.ReadBackfillProcesses(context.Background()) @@ -270,7 +271,7 @@ func TestArchiver_LatestStopsAtExistingBlock(t *testing.T) { BeaconBlockHash: blobtest.Three, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Three.String()], + Data: beacon.SidecarsByBlock[blobtest.Three.String()], }, }) @@ -283,17 +284,17 @@ func TestArchiver_LatestStopsAtExistingBlock(t *testing.T) { fs.CheckExistsOrFail(t, blobtest.Five) five := fs.ReadOrFail(t, blobtest.Five) require.Equal(t, five.Header.BeaconBlockHash, blobtest.Five) - require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()]) + require.Equal(t, five.BlobSidecars.Data, beacon.SidecarsByBlock[blobtest.Five.String()]) fs.CheckExistsOrFail(t, blobtest.Four) four := fs.ReadOrFail(t, blobtest.Four) require.Equal(t, four.Header.BeaconBlockHash, blobtest.Four) - require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()]) + require.Equal(t, five.BlobSidecars.Data, beacon.SidecarsByBlock[blobtest.Five.String()]) fs.CheckExistsOrFail(t, blobtest.Three) three := fs.ReadOrFail(t, blobtest.Three) require.Equal(t, three.Header.BeaconBlockHash, blobtest.Three) - require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()]) + require.Equal(t, five.BlobSidecars.Data, beacon.SidecarsByBlock[blobtest.Five.String()]) } func TestArchiver_LatestNoNewData(t *testing.T) { @@ -306,7 +307,7 @@ func TestArchiver_LatestNoNewData(t *testing.T) { BeaconBlockHash: common.Hash(beacon.Headers["head"].Root), }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Three.String()], + Data: beacon.SidecarsByBlock[blobtest.Three.String()], }, }) @@ -330,7 +331,7 @@ func TestArchiver_LatestConsumesNewBlocks(t *testing.T) { BeaconBlockHash: common.Hash(beacon.Headers[blobtest.Four.String()].Root), }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Four.String()], + Data: beacon.SidecarsByBlock[blobtest.Four.String()], }, }) @@ -360,7 +361,7 @@ func TestArchiver_LatestStopsAtOrigin(t *testing.T) { BeaconBlockHash: blobtest.OriginBlock, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.OriginBlock.String()], + Data: beacon.SidecarsByBlock[blobtest.OriginBlock.String()], }, }) @@ -375,7 +376,7 @@ func TestArchiver_LatestStopsAtOrigin(t *testing.T) { for _, hash := range toWrite { fs.CheckExistsOrFail(t, hash) data := fs.ReadOrFail(t, hash) - require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[hash.String()]) + require.Equal(t, data.BlobSidecars.Data, beacon.SidecarsByBlock[hash.String()]) } } @@ -389,7 +390,7 @@ func TestArchiver_LatestRetriesOnFailure(t *testing.T) { BeaconBlockHash: blobtest.Three, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Three.String()], + Data: beacon.SidecarsByBlock[blobtest.Three.String()], }, }) @@ -416,7 +417,7 @@ func TestArchiver_LatestHaltsOnPersistentError(t *testing.T) { BeaconBlockHash: blobtest.Three, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Three.String()], + Data: beacon.SidecarsByBlock[blobtest.Three.String()], }, }) @@ -443,7 +444,7 @@ func TestArchiver_RearchiveRange(t *testing.T) { BeaconBlockHash: blobtest.Three, }, BlobSidecars: storage.BlobSidecars{ - Data: beacon.Blobs[blobtest.Three.String()], + Data: beacon.SidecarsByBlock[blobtest.Three.String()], }, }) @@ -454,7 +455,7 @@ func TestArchiver_RearchiveRange(t *testing.T) { fs.CheckNotExistsOrFail(t, blobtest.Four) // this modifies the blobs at 3, purely to test the blob is rearchived - beacon.Blobs[blobtest.Three.String()] = blobtest.NewBlobSidecars(t, 6) + beacon.SidecarsByBlock[blobtest.Three.String()] = blobtest.NewBlobSidecars(t, 6, beacon.Headers[blobtest.Three.String()].Header) from, to := blobtest.StartSlot+1, blobtest.StartSlot+4 @@ -471,5 +472,49 @@ func TestArchiver_RearchiveRange(t *testing.T) { fs.CheckExistsOrFail(t, blobtest.Four) // Should have overwritten any existing blobs - require.Equal(t, fs.ReadOrFail(t, blobtest.Three).BlobSidecars.Data, beacon.Blobs[blobtest.Three.String()]) + require.Equal(t, fs.ReadOrFail(t, blobtest.Three).BlobSidecars.Data, beacon.SidecarsByBlock[blobtest.Three.String()]) +} + +func TestArchiver_FetchBlobs_FallbackToSidecars(t *testing.T) { + beacon := beacontest.NewDefaultStubBeaconClient(t) + beacon.FailBlobs = true // Make blobs endpoint fail + svc, fs := setup(t, beacon) + + fs.CheckNotExistsOrFail(t, blobtest.One) + + // Should fall back to blob sidecars endpoint + header, alreadyExists, err := svc.persistBlobsForBlockToS3(context.Background(), blobtest.One.String(), false) + require.False(t, alreadyExists) + require.NoError(t, err) + require.NotNil(t, header) + + fs.CheckExistsOrFail(t, blobtest.One) + stored := fs.ReadOrFail(t, blobtest.One) + require.Equal(t, len(beacon.SidecarsByBlock[blobtest.One.String()]), len(stored.BlobSidecars.Data)) +} + +func TestArchiver_FetchBlobs_Success(t *testing.T) { + beacon := beacontest.NewDefaultStubBeaconClient(t) + // FailBlobs = false (default) - blobs endpoint should succeed + svc, fs := setup(t, beacon) + + fs.CheckNotExistsOrFail(t, blobtest.Two) + + // Should use blobs endpoint successfully + header, alreadyExists, err := svc.persistBlobsForBlockToS3(context.Background(), blobtest.Two.String(), false) + require.False(t, alreadyExists) + require.NoError(t, err) + require.NotNil(t, header) + + fs.CheckExistsOrFail(t, blobtest.Two) + stored := fs.ReadOrFail(t, blobtest.Two) + // Should have stored the correct number of blobs + require.Equal(t, len(beacon.SidecarsByBlock[blobtest.Two.String()]), len(stored.BlobSidecars.Data)) + + // Verify that KZG commitments and proofs were derived correctly + for i, storedSidecar := range stored.BlobSidecars.Data { + originalSidecar := beacon.SidecarsByBlock[blobtest.Two.String()][i] + // The blob data should match + require.Equal(t, originalSidecar.Blob, storedSidecar.Blob) + } } diff --git a/common/beacon/beacontest/stub.go b/common/beacon/beacontest/stub.go index 0f1030e..f3f575a 100644 --- a/common/beacon/beacontest/stub.go +++ b/common/beacon/beacontest/stub.go @@ -15,8 +15,9 @@ import ( ) type StubBeaconClient struct { - Headers map[string]*v1.BeaconBlockHeader - Blobs map[string][]*deneb.BlobSidecar + Headers map[string]*v1.BeaconBlockHeader + SidecarsByBlock map[string][]*deneb.BlobSidecar + FailBlobs bool } func (s *StubBeaconClient) BeaconBlockHeader(ctx context.Context, opts *api.BeaconBlockHeaderOpts) (*api.Response[*v1.BeaconBlockHeader], error) { @@ -30,7 +31,7 @@ func (s *StubBeaconClient) BeaconBlockHeader(ctx context.Context, opts *api.Beac } func (s *StubBeaconClient) BlobSidecars(ctx context.Context, opts *api.BlobSidecarsOpts) (*api.Response[[]*deneb.BlobSidecar], error) { - blobs, found := s.Blobs[opts.Block] + blobs, found := s.SidecarsByBlock[opts.Block] if !found { return nil, fmt.Errorf("block not found") } @@ -39,10 +40,31 @@ func (s *StubBeaconClient) BlobSidecars(ctx context.Context, opts *api.BlobSidec }, nil } +// Blobs implements the BlobsProvider interface, converting sidecars to blobs +func (s *StubBeaconClient) Blobs(ctx context.Context, opts *api.BlobsOpts) (*api.Response[v1.Blobs], error) { + if s.FailBlobs { + return nil, fmt.Errorf("blobs endpoint unavailable") + } + + sidecars, found := s.SidecarsByBlock[opts.Block] + if !found { + return nil, fmt.Errorf("block not found") + } + + blobs := make(v1.Blobs, len(sidecars)) + for i, sidecar := range sidecars { + blobs[i] = &sidecar.Blob + } + + return &api.Response[v1.Blobs]{ + Data: blobs, + }, nil +} + func NewEmptyStubBeaconClient() *StubBeaconClient { return &StubBeaconClient{ - Headers: make(map[string]*v1.BeaconBlockHeader), - Blobs: make(map[string][]*deneb.BlobSidecar), + Headers: make(map[string]*v1.BeaconBlockHeader), + SidecarsByBlock: make(map[string][]*deneb.BlobSidecar), } } @@ -61,22 +83,31 @@ func NewDefaultStubBeaconClient(t *testing.T) *StubBeaconClient { startSlot := blobtest.StartSlot - originBlobs := blobtest.NewBlobSidecars(t, 1) - oneBlobs := blobtest.NewBlobSidecars(t, 2) - twoBlobs := blobtest.NewBlobSidecars(t, 0) - threeBlobs := blobtest.NewBlobSidecars(t, 4) - fourBlobs := blobtest.NewBlobSidecars(t, 5) - fiveBlobs := blobtest.NewBlobSidecars(t, 6) + // Create headers first so they can be used for blobs + originHeader := makeHeader(startSlot, blobtest.OriginBlock, common.Hash{9, 9, 9}) + oneHeader := makeHeader(startSlot+1, blobtest.One, blobtest.OriginBlock) + twoHeader := makeHeader(startSlot+2, blobtest.Two, blobtest.One) + threeHeader := makeHeader(startSlot+3, blobtest.Three, blobtest.Two) + fourHeader := makeHeader(startSlot+4, blobtest.Four, blobtest.Three) + fiveHeader := makeHeader(startSlot+5, blobtest.Five, blobtest.Four) + + // Create blobs with valid headers + originBlobs := blobtest.NewBlobSidecars(t, 1, originHeader.Header) + oneBlobs := blobtest.NewBlobSidecars(t, 2, oneHeader.Header) + twoBlobs := blobtest.NewBlobSidecars(t, 0, twoHeader.Header) + threeBlobs := blobtest.NewBlobSidecars(t, 4, threeHeader.Header) + fourBlobs := blobtest.NewBlobSidecars(t, 5, fourHeader.Header) + fiveBlobs := blobtest.NewBlobSidecars(t, 6, fiveHeader.Header) return &StubBeaconClient{ Headers: map[string]*v1.BeaconBlockHeader{ // Lookup by hash - blobtest.OriginBlock.String(): makeHeader(startSlot, blobtest.OriginBlock, common.Hash{9, 9, 9}), - blobtest.One.String(): makeHeader(startSlot+1, blobtest.One, blobtest.OriginBlock), - blobtest.Two.String(): makeHeader(startSlot+2, blobtest.Two, blobtest.One), - blobtest.Three.String(): makeHeader(startSlot+3, blobtest.Three, blobtest.Two), - blobtest.Four.String(): makeHeader(startSlot+4, blobtest.Four, blobtest.Three), - blobtest.Five.String(): makeHeader(startSlot+5, blobtest.Five, blobtest.Four), + blobtest.OriginBlock.String(): originHeader, + blobtest.One.String(): oneHeader, + blobtest.Two.String(): twoHeader, + blobtest.Three.String(): threeHeader, + blobtest.Four.String(): fourHeader, + blobtest.Five.String(): fiveHeader, // Lookup by identifier "head": makeHeader(startSlot+5, blobtest.Five, blobtest.Four), @@ -90,7 +121,7 @@ func NewDefaultStubBeaconClient(t *testing.T) *StubBeaconClient { strconv.FormatUint(startSlot+4, 10): makeHeader(startSlot+4, blobtest.Four, blobtest.Three), strconv.FormatUint(startSlot+5, 10): makeHeader(startSlot+5, blobtest.Five, blobtest.Four), }, - Blobs: map[string][]*deneb.BlobSidecar{ + SidecarsByBlock: map[string][]*deneb.BlobSidecar{ // Lookup by hash blobtest.OriginBlock.String(): originBlobs, blobtest.One.String(): oneBlobs, diff --git a/common/beacon/client.go b/common/beacon/client.go index eeea770..f433dad 100644 --- a/common/beacon/client.go +++ b/common/beacon/client.go @@ -13,6 +13,7 @@ import ( type Client interface { client.BeaconBlockHeadersProvider client.BlobSidecarsProvider + client.BlobsProvider } // NewBeaconClient returns a new HTTP beacon client. diff --git a/common/blobtest/helpers.go b/common/blobtest/helpers.go index 57d9523..3f1da32 100644 --- a/common/blobtest/helpers.go +++ b/common/blobtest/helpers.go @@ -2,14 +2,24 @@ package blobtest import ( "crypto/rand" + "math/big" "testing" "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/phase0" + gokzg4844 "github.com/crate-crypto/go-kzg-4844" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" ) +// BLS12-381 field modulus (as a big-endian byte array) +// This is the prime field modulus used by BLS12-381 +// In decimal: 52435875175126190479447740508185965837690552500527637822603658699938581184513 +var fieldModulus = [32]byte{ + 0x73, 0xed, 0xa7, 0x53, 0x29, 0xd7, 0xd4, 0x69, 0x5e, 0x7f, 0x6e, 0x0f, 0xf9, 0xd7, 0xfb, 0xd8, + 0xc4, 0xb9, 0x35, 0x6d, 0x47, 0x19, 0xb7, 0x01, 0x8b, 0x0c, 0x6c, 0x6f, 0xd9, 0x52, 0x53, 0x73, +} + var ( OriginBlock = common.Hash{9, 9, 9, 9, 9} One = common.Hash{1} @@ -31,22 +41,67 @@ func RandBytes(t *testing.T, size uint) []byte { return randomBytes } -func NewBlobSidecar(t *testing.T, i uint) *deneb.BlobSidecar { - return &deneb.BlobSidecar{ - Index: deneb.BlobIndex(i), - Blob: deneb.Blob(RandBytes(t, 131072)), - KZGCommitment: deneb.KZGCommitment(RandBytes(t, 48)), - KZGProof: deneb.KZGProof(RandBytes(t, 48)), - SignedBlockHeader: &phase0.SignedBeaconBlockHeader{ +// NewBlobSidecars creates blob sidecars with valid KZG commitments and proofs. +// This generates blob data that respects BLS12-381 field constraints: each 32-byte +// chunk must be less than the field modulus to be a valid field element. +// The blob data is cryptographically valid and will work with KZG operations. +// If signedHeader is provided, it will be used for all sidecars; otherwise an empty header is created. +func NewBlobSidecars(t *testing.T, count uint, signedHeader *phase0.SignedBeaconBlockHeader) []*deneb.BlobSidecar { + kzgCtx, err := gokzg4844.NewContext4096Secure() + require.NoError(t, err) + + // BLS12-381 field modulus as a big.Int + fieldModulusBig := new(big.Int).SetBytes(fieldModulus[:]) + + // Use provided header or create empty one + if signedHeader == nil { + signedHeader = &phase0.SignedBeaconBlockHeader{ Message: &phase0.BeaconBlockHeader{}, - }, + } } -} -func NewBlobSidecars(t *testing.T, count uint) []*deneb.BlobSidecar { result := make([]*deneb.BlobSidecar, count) for i := uint(0); i < count; i++ { - result[i] = NewBlobSidecar(t, i) + var blob deneb.Blob + + // Generate blob data where each 32-byte field element is less than the field modulus + // This ensures valid KZG operations + for j := 0; j < len(blob); j += 32 { + // Generate random bytes for this field element + randomBytes := RandBytes(t, 32) + + // Convert to big.Int and reduce modulo field modulus + randomBig := new(big.Int).SetBytes(randomBytes) + reduced := new(big.Int).Mod(randomBig, fieldModulusBig) + + // Convert back to 32-byte representation (zero-padded) + reducedBytes := reduced.Bytes() + if len(reducedBytes) < 32 { + // Pad with leading zeros + paddedBytes := make([]byte, 32) + copy(paddedBytes[32-len(reducedBytes):], reducedBytes) + copy(blob[j:j+32], paddedBytes[:]) + } else { + copy(blob[j:j+32], reducedBytes[:32]) + } + } + + // Now compute KZG commitment from this valid blob data + kzgBlob := (*gokzg4844.Blob)(&blob) + commitment, err := kzgCtx.BlobToKZGCommitment(kzgBlob, 0) + require.NoError(t, err, "failed to compute KZG commitment for blob %d", i) + + // Compute KZG proof + proof, err := kzgCtx.ComputeBlobKZGProof(kzgBlob, commitment, 0) + require.NoError(t, err, "failed to compute KZG proof for blob %d", i) + + result[i] = &deneb.BlobSidecar{ + Index: deneb.BlobIndex(i), + Blob: blob, + KZGCommitment: deneb.KZGCommitment(commitment), + KZGProof: deneb.KZGProof(proof), + SignedBlockHeader: signedHeader, + } } return result } diff --git a/go.mod b/go.mod index 9343bd3..b134d8c 100644 --- a/go.mod +++ b/go.mod @@ -97,3 +97,5 @@ require ( ) replace github.com/ethereum/go-ethereum v1.101500.1 => github.com/ethereum-optimism/op-geth v1.101500.1 + +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20251008201737-cffbe3a2917c diff --git a/go.sum b/go.sum index b622399..b3264ca 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= -github.com/attestantio/go-eth2-client v0.27.1 h1:g7bm+gG/p+gfzYdEuxuAepVWYb8EO+2KojV5/Lo2BxM= -github.com/attestantio/go-eth2-client v0.27.1/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= @@ -196,6 +194,8 @@ github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1 github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= github.com/pk910/dynamic-ssz v0.0.4 h1:DT29+1055tCEPCaR4V/ez+MOKW7BzBsmjyFvBRqx0ME= github.com/pk910/dynamic-ssz v0.0.4/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= +github.com/pk910/go-eth2-client v0.0.0-20251008201737-cffbe3a2917c h1:+S57E+tGtH263WMa5u8L1F0mtL9CCaUax34J1UFz39s= +github.com/pk910/go-eth2-client v0.0.0-20251008201737-cffbe3a2917c/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/validator/service/service_test.go b/validator/service/service_test.go index da72001..a68e366 100644 --- a/validator/service/service_test.go +++ b/validator/service/service_test.go @@ -34,7 +34,7 @@ type stubBlobSidecarClient struct { // setResponses configures the stub to return the same data as the beacon client for all FetchSidecars invocations func (s *stubBlobSidecarClient) setResponses(sbc *beacontest.StubBeaconClient) { - for k, v := range sbc.Blobs { + for k, v := range sbc.SidecarsByBlock { s.data[k] = response{ data: storage.BlobSidecars{Data: v}, err: nil, @@ -129,7 +129,7 @@ func TestValidatorService_CompletelyDifferentBlobData(t *testing.T) { beacon.setResponses(headers) blob.setResponses(headers) blob.setResponse(blockOne, 200, storage.BlobSidecars{ - Data: blobtest.NewBlobSidecars(t, 1), + Data: blobtest.NewBlobSidecars(t, 1, nil), }, nil) result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.EndSlot)) @@ -193,7 +193,7 @@ func TestValidatorService_MistmatchedBlobFields(t *testing.T) { blob.setResponses(headers) // Deep copy the blob data - d, err := json.Marshal(headers.Blobs[blockOne]) + d, err := json.Marshal(headers.SidecarsByBlock[blockOne]) require.NoError(t, err) var c []*deneb.BlobSidecar err = json.Unmarshal(d, &c) From d41cfb58cb3c09cf533cf2e086284005ad8dee5c Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Wed, 15 Oct 2025 23:29:05 -0500 Subject: [PATCH 3/5] refactor: prefer blob_sidecars endpoint to avoid recomputing KZG proofs Switch to prioritizing /eth/v1/beacon/blob_sidecars over /eth/v1/beacon/blobs since the sidecars endpoint provides KZG commitments and proofs directly, avoiding unnecessary computation. --- archiver/service/archiver.go | 45 ++++++++++++++++++------------- archiver/service/archiver_test.go | 14 +++++----- common/beacon/beacontest/stub.go | 10 +++---- 3 files changed, 38 insertions(+), 31 deletions(-) diff --git a/archiver/service/archiver.go b/archiver/service/archiver.go index 487d79d..ce0781c 100644 --- a/archiver/service/archiver.go +++ b/archiver/service/archiver.go @@ -125,8 +125,9 @@ func (a *Archiver) Stop(ctx context.Context) error { // perform any validation of the blobs, it assumes a trusted beacon node. See: // https://github.com/base/blob-archiver/issues/4. // -// The function prefers the /eth/v1/beacon/blobs endpoint but falls back to /eth/v1/beacon/blob_sidecars -// if the blobs endpoint fails. +// The function prefers the /eth/v1/beacon/blob_sidecars endpoint but falls back to /eth/v1/beacon/blobs +// if the blob sidecars endpoint fails. This avoids recomputing KZG commitments and proofs when they're +// available from the beacon node. func (a *Archiver) persistBlobsForBlockToS3(ctx context.Context, blockIdentifier string, overwrite bool) (*v1.BeaconBlockHeader, bool, error) { currentHeader, err := a.beaconClient.BeaconBlockHeader(ctx, &api.BeaconBlockHeaderOpts{ Block: blockIdentifier, @@ -148,38 +149,44 @@ func (a *Archiver) persistBlobsForBlockToS3(ctx context.Context, blockIdentifier return currentHeader.Data, true, nil } - // Try the new blobs endpoint first + // Try the blob sidecars endpoint first to get commitments and proofs directly var blobSidecarData []*deneb.BlobSidecar - blobs, err := a.beaconClient.Blobs(ctx, &api.BlobsOpts{ + blobSidecarsResp, err := a.beaconClient.BlobSidecars(ctx, &api.BlobSidecarsOpts{ Block: currentHeader.Data.Root.String(), }) - if err == nil && blobs != nil && blobs.Data != nil && len(blobs.Data) > 0 { - // Successfully fetched blobs, compute commitments and proofs from blob data - a.log.Debug("fetched blobs from blobs endpoint, computing KZG commitments and proofs", "count", len(blobs.Data)) - - var err error - blobSidecarData, err = blobsToSidecars(blobs.Data, currentHeader.Data) - if err != nil { - a.log.Error("failed to compute KZG commitments and proofs for blobs", "err", err) - return nil, false, err - } + if err == nil && blobSidecarsResp != nil && blobSidecarsResp.Data != nil { + // Successfully fetched blob sidecars with KZG commitments and proofs + a.log.Debug("fetched blob sidecars from blob_sidecars endpoint", "count", len(blobSidecarsResp.Data)) + blobSidecarData = blobSidecarsResp.Data } else { - // Fall back to blob sidecars endpoint + // Fall back to blobs endpoint and compute commitments and proofs if err != nil { - a.log.Debug("blobs endpoint failed, falling back to blob sidecars", "err", err) + a.log.Debug("blob sidecars endpoint failed, falling back to blobs", "err", err) } - blobSidecarsResp, fallbackErr := a.beaconClient.BlobSidecars(ctx, &api.BlobSidecarsOpts{ + blobs, fallbackErr := a.beaconClient.Blobs(ctx, &api.BlobsOpts{ Block: currentHeader.Data.Root.String(), }) if fallbackErr != nil { - a.log.Error("failed to fetch blob sidecars", "err", fallbackErr) + a.log.Error("failed to fetch blobs", "err", fallbackErr) return nil, false, fallbackErr } - blobSidecarData = blobSidecarsResp.Data + if blobs == nil || blobs.Data == nil { + a.log.Error("blobs endpoint returned nil data") + return nil, false, errors.New("blobs endpoint returned nil data") + } + + a.log.Debug("fetched blobs from blobs endpoint, computing KZG commitments and proofs", "count", len(blobs.Data)) + + var computeErr error + blobSidecarData, computeErr = blobsToSidecars(blobs.Data, currentHeader.Data) + if computeErr != nil { + a.log.Error("failed to compute KZG commitments and proofs for blobs", "err", computeErr) + return nil, false, computeErr + } } a.log.Debug("fetched blob sidecars", "count", len(blobSidecarData)) diff --git a/archiver/service/archiver_test.go b/archiver/service/archiver_test.go index 680a16f..db828d4 100644 --- a/archiver/service/archiver_test.go +++ b/archiver/service/archiver_test.go @@ -475,14 +475,14 @@ func TestArchiver_RearchiveRange(t *testing.T) { require.Equal(t, fs.ReadOrFail(t, blobtest.Three).BlobSidecars.Data, beacon.SidecarsByBlock[blobtest.Three.String()]) } -func TestArchiver_FetchBlobs_FallbackToSidecars(t *testing.T) { +func TestArchiver_FetchBlobSidecars_Success(t *testing.T) { beacon := beacontest.NewDefaultStubBeaconClient(t) - beacon.FailBlobs = true // Make blobs endpoint fail + // FailSidecars = false (default) - blob sidecars endpoint should succeed svc, fs := setup(t, beacon) fs.CheckNotExistsOrFail(t, blobtest.One) - // Should fall back to blob sidecars endpoint + // Should use blob sidecars endpoint successfully header, alreadyExists, err := svc.persistBlobsForBlockToS3(context.Background(), blobtest.One.String(), false) require.False(t, alreadyExists) require.NoError(t, err) @@ -493,14 +493,14 @@ func TestArchiver_FetchBlobs_FallbackToSidecars(t *testing.T) { require.Equal(t, len(beacon.SidecarsByBlock[blobtest.One.String()]), len(stored.BlobSidecars.Data)) } -func TestArchiver_FetchBlobs_Success(t *testing.T) { +func TestArchiver_FetchBlobs_FallbackToBlobs(t *testing.T) { beacon := beacontest.NewDefaultStubBeaconClient(t) - // FailBlobs = false (default) - blobs endpoint should succeed + beacon.FailSidecars = true // Make blob sidecars endpoint fail svc, fs := setup(t, beacon) fs.CheckNotExistsOrFail(t, blobtest.Two) - // Should use blobs endpoint successfully + // Should fall back to blobs endpoint and compute KZG commitments/proofs header, alreadyExists, err := svc.persistBlobsForBlockToS3(context.Background(), blobtest.Two.String(), false) require.False(t, alreadyExists) require.NoError(t, err) @@ -511,7 +511,7 @@ func TestArchiver_FetchBlobs_Success(t *testing.T) { // Should have stored the correct number of blobs require.Equal(t, len(beacon.SidecarsByBlock[blobtest.Two.String()]), len(stored.BlobSidecars.Data)) - // Verify that KZG commitments and proofs were derived correctly + // Verify that KZG commitments and proofs were computed correctly from blob data for i, storedSidecar := range stored.BlobSidecars.Data { originalSidecar := beacon.SidecarsByBlock[blobtest.Two.String()][i] // The blob data should match diff --git a/common/beacon/beacontest/stub.go b/common/beacon/beacontest/stub.go index f3f575a..b6bd496 100644 --- a/common/beacon/beacontest/stub.go +++ b/common/beacon/beacontest/stub.go @@ -17,7 +17,7 @@ import ( type StubBeaconClient struct { Headers map[string]*v1.BeaconBlockHeader SidecarsByBlock map[string][]*deneb.BlobSidecar - FailBlobs bool + FailSidecars bool } func (s *StubBeaconClient) BeaconBlockHeader(ctx context.Context, opts *api.BeaconBlockHeaderOpts) (*api.Response[*v1.BeaconBlockHeader], error) { @@ -31,6 +31,10 @@ func (s *StubBeaconClient) BeaconBlockHeader(ctx context.Context, opts *api.Beac } func (s *StubBeaconClient) BlobSidecars(ctx context.Context, opts *api.BlobSidecarsOpts) (*api.Response[[]*deneb.BlobSidecar], error) { + if s.FailSidecars { + return nil, fmt.Errorf("blob sidecars endpoint unavailable") + } + blobs, found := s.SidecarsByBlock[opts.Block] if !found { return nil, fmt.Errorf("block not found") @@ -42,10 +46,6 @@ func (s *StubBeaconClient) BlobSidecars(ctx context.Context, opts *api.BlobSidec // Blobs implements the BlobsProvider interface, converting sidecars to blobs func (s *StubBeaconClient) Blobs(ctx context.Context, opts *api.BlobsOpts) (*api.Response[v1.Blobs], error) { - if s.FailBlobs { - return nil, fmt.Errorf("blobs endpoint unavailable") - } - sidecars, found := s.SidecarsByBlock[opts.Block] if !found { return nil, fmt.Errorf("block not found") From 271b3e3d925e8721dde140e9dc9ead8b553af897 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Thu, 16 Oct 2025 15:11:14 -0500 Subject: [PATCH 4/5] chore: update go.mod to mark go-kzg-4844 as direct dependency The go-kzg-4844 package is now directly used in the codebase after migrating to the beacon blobs endpoint, so it should be listed as a direct dependency rather than an indirect one. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b134d8c..ef081a0 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.24.0 require ( github.com/attestantio/go-eth2-client v0.27.1 + github.com/crate-crypto/go-kzg-4844 v1.1.0 github.com/ethereum-optimism/optimism v1.11.2 github.com/ethereum/go-ethereum v1.101500.1 github.com/go-chi/chi/v5 v5.0.12 @@ -25,7 +26,6 @@ require ( github.com/consensys/gnark-crypto v0.16.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect - github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect From 4b122092a8128970d911bbd1faa29114faaef942 Mon Sep 17 00:00:00 2001 From: Niran Babalola Date: Thu, 16 Oct 2025 15:37:47 -0500 Subject: [PATCH 5/5] fix: use canonical BLS modulus from go-kzg-4844 Replaced custom field modulus constant with gokzg4844.BlsModulus, which is the canonical BLS12-381 scalar field modulus from our existing dependency. This fixes an incorrect modulus value and provides a single source of truth for the BLS_MODULUS constant as specified in EIP-4844. The go-kzg-4844 library already exports this value as BlsModulus, so there's no need to duplicate it. --- common/blobtest/helpers.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/common/blobtest/helpers.go b/common/blobtest/helpers.go index 3f1da32..5591b25 100644 --- a/common/blobtest/helpers.go +++ b/common/blobtest/helpers.go @@ -12,14 +12,6 @@ import ( "github.com/stretchr/testify/require" ) -// BLS12-381 field modulus (as a big-endian byte array) -// This is the prime field modulus used by BLS12-381 -// In decimal: 52435875175126190479447740508185965837690552500527637822603658699938581184513 -var fieldModulus = [32]byte{ - 0x73, 0xed, 0xa7, 0x53, 0x29, 0xd7, 0xd4, 0x69, 0x5e, 0x7f, 0x6e, 0x0f, 0xf9, 0xd7, 0xfb, 0xd8, - 0xc4, 0xb9, 0x35, 0x6d, 0x47, 0x19, 0xb7, 0x01, 0x8b, 0x0c, 0x6c, 0x6f, 0xd9, 0x52, 0x53, 0x73, -} - var ( OriginBlock = common.Hash{9, 9, 9, 9, 9} One = common.Hash{1} @@ -50,8 +42,9 @@ func NewBlobSidecars(t *testing.T, count uint, signedHeader *phase0.SignedBeacon kzgCtx, err := gokzg4844.NewContext4096Secure() require.NoError(t, err) - // BLS12-381 field modulus as a big.Int - fieldModulusBig := new(big.Int).SetBytes(fieldModulus[:]) + // Use the BLS12-381 scalar field modulus from go-kzg-4844 + // This is the order of the G1/G2 subgroups, used to validate blob field elements + fieldModulusBig := new(big.Int).SetBytes(gokzg4844.BlsModulus[:]) // Use provided header or create empty one if signedHeader == nil {