From ab6399bb64e0d8e6a7093d1871440927587629b1 Mon Sep 17 00:00:00 2001 From: Urvi Date: Fri, 12 Sep 2025 20:43:21 +0530 Subject: [PATCH 1/2] Update CDP code samples using BufferedStorageBackend --- .../apps/ingest-sdk/ingestion-pipeline-code.mdx | 10 +++------- .../ledgerbackends/bufferedstoragebackend.mdx | 14 +++++++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/build/apps/ingest-sdk/ingestion-pipeline-code.mdx b/docs/build/apps/ingest-sdk/ingestion-pipeline-code.mdx index e6e13f3526..49c8d36fe7 100644 --- a/docs/build/apps/ingest-sdk/ingestion-pipeline-code.mdx +++ b/docs/build/apps/ingest-sdk/ingestion-pipeline-code.mdx @@ -173,13 +173,13 @@ func (adapter *LedgerMetadataInboundAdapter) Run(ctx context.Context) error { ledgerRange := ledgerbackend.UnboundedRange(latestNetworkLedger) - pubConfig := PublisherConfig{ + pubConfig := ingest.PublisherConfig{ DataStoreConfig: adapter.dataStoreConfig, - BufferedStorageConfig: DefaultBufferedStorageBackendConfig(adapter.dataStoreConfig.Schema.LedgersPerFile), + BufferedStorageConfig: ingest.DefaultBufferedStorageBackendConfig(1), } fmt.Printf("beginning payments stream, starting at ledger %v ...\n", latestNetworkLedger) - return ApplyLedgerMetadata(ledgerRange, pubConfig, ctx, + return ingest.ApplyLedgerMetadata(ledgerRange, pubConfig, ctx, func(lcm xdr.LedgerCloseMeta) error { for _, processor := range adapter.processors { if err = processor.Process(ctx, Message{Payload: lcm}); err != nil { @@ -249,10 +249,6 @@ type = "GCS" [params] destination_bucket_path = "my-gcs-bucketname/prefix1/prefix2" -[schema] -ledgers_per_file = 1 -files_per_partition = 64000 - ``` diff --git a/docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx b/docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx index 53f91a8140..90445ad414 100644 --- a/docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx +++ b/docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx @@ -109,10 +109,6 @@ func main() { Params: map[string]string{ "destination_bucket_path": "your-gcs-bucket/data", // Replace with actual GCS bucket path }, - Schema: datastore.DataStoreSchema{ - LedgersPerFile: 1, // Defines how many ledgers per file - FilesPerPartition: 64000, // Defines partition size - }, } // Initialize the datastore @@ -130,8 +126,16 @@ func main() { RetryWait: 5 * time.Second, // Wait time between retries } + // Per SEP-54 (https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0054.md), + // the schema is stored in the datastore; LoadSchema retrieves it. + var schema datastore.DataStoreSchema + schema, err = datastore.LoadSchema(context.Background(), dataStore, datastoreConfig) + if err != nil { + log.Fatal(errors.Wrap(err, "failed to retrieve datastore schema")) + } + // Initialize the backend - backend, err := ledgerbackend.NewBufferedStorageBackend(backendConfig, dataStore) + backend, err := ledgerbackend.NewBufferedStorageBackend(backendConfig, dataStore, schema) if err != nil { log.Fatal(errors.Wrap(err, "failed to create buffered storage backend")) } From 84561471d53ea166cb68969f208933d34aa173a3 Mon Sep 17 00:00:00 2001 From: urvisavla Date: Fri, 12 Sep 2025 23:18:16 +0530 Subject: [PATCH 2/2] Update docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx Co-authored-by: tamirms --- .../developer_guide/ledgerbackends/bufferedstoragebackend.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx b/docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx index 90445ad414..5fcfc8fa66 100644 --- a/docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx +++ b/docs/data/indexers/build-your-own/ingest-sdk/developer_guide/ledgerbackends/bufferedstoragebackend.mdx @@ -128,8 +128,7 @@ func main() { // Per SEP-54 (https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0054.md), // the schema is stored in the datastore; LoadSchema retrieves it. - var schema datastore.DataStoreSchema - schema, err = datastore.LoadSchema(context.Background(), dataStore, datastoreConfig) + schema, err := datastore.LoadSchema(context.Background(), dataStore, datastoreConfig) if err != nil { log.Fatal(errors.Wrap(err, "failed to retrieve datastore schema")) }