From 9dffeb47246d278977406f79683e55ffdbdb83a6 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 6 Nov 2025 15:25:18 -0800 Subject: [PATCH 01/17] perf --- lib/ocrypto/aes_gcm.go | 16 ++++++++++++++++ sdk/experimental/tdf/writer.go | 19 +++++++++---------- sdk/internal/zipstream/segment_writer.go | 10 +++++++--- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/lib/ocrypto/aes_gcm.go b/lib/ocrypto/aes_gcm.go index 77aaf647ea..044be7f65b 100644 --- a/lib/ocrypto/aes_gcm.go +++ b/lib/ocrypto/aes_gcm.go @@ -46,6 +46,22 @@ func (aesGcm AesGcm) Encrypt(data []byte) ([]byte, error) { cipherText := gcm.Seal(nonce, nonce, data, nil) return cipherText, nil } +func (aesGcm AesGcm) EncryptWithDestination(data []byte, dst []byte) ([]byte, error) { + nonce, err := RandomBytes(GcmStandardNonceSize) + if err != nil { + return nil, err + } + copy(dst[:GcmStandardNonceSize], nonce) + + + gcm, err := cipher.NewGCMWithNonceSize(aesGcm.block, GcmStandardNonceSize) + if err != nil { + return nil, fmt.Errorf("cipher.NewGCMWithNonceSize failed: %w", err) + } + + cipherText := gcm.Seal(dst[:GcmStandardNonceSize], nonce, data, nil) + return cipherText, nil +} // EncryptWithIV encrypts data with symmetric key. // NOTE: This method use default auth tag as aes block size(16 bytes) diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index ed7b12d97e..3174c01ba1 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - "hash/crc32" "log/slog" "sort" "sync" @@ -232,34 +231,35 @@ func NewWriter(_ context.Context, opts ...Option[*WriterConfig]) (*Writer, error // uploadToS3(segment1, "part-001") func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*SegmentResult, error) { w.mutex.Lock() - defer w.mutex.Unlock() if w.finalized { + w.mutex.Unlock() return nil, ErrAlreadyFinalized } if index < 0 { + w.mutex.Unlock() return nil, ErrInvalidSegmentIndex } // Check for duplicate segments using map lookup if _, exists := w.segments[index]; exists { + w.mutex.Unlock() return nil, ErrSegmentAlreadyWritten } if index > w.maxSegmentIndex { w.maxSegmentIndex = index } + w.mutex.Unlock() - // Calculate CRC32 before encryption for integrity tracking - crc32Checksum := crc32.ChecksumIEEE(data) - + segmentBlock := make([]byte, 100+len(data)+ 100) // Encrypt directly without unnecessary copying - the archive layer will handle copying if needed - segmentCipher, err := w.block.Encrypt(data) + segmentCipher, err := w.block.EncryptWithDestination(data, segmentBlock[:100]) if err != nil { return nil, err } - + segmentBlock = segmentBlock[:100+len(segmentCipher)] segmentSig, err := calculateSignature(segmentCipher, w.dek, w.segmentIntegrityAlgorithm, false) // Don't ever hex encode new tdf's if err != nil { return nil, err @@ -272,7 +272,7 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg EncryptedSize: int64(len(segmentCipher)), } - zipBytes, err := w.archiveWriter.WriteSegment(ctx, index, segmentCipher) + zipBytes, err := w.archiveWriter.WriteSegment(ctx, index, segmentBlock) if err != nil { return nil, err } @@ -282,8 +282,7 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg Index: index, Hash: segmentHash, PlaintextSize: int64(len(data)), - EncryptedSize: int64(len(segmentCipher)), - CRC32: crc32Checksum, + EncryptedSize: int64(len(zipBytes)), }, nil } diff --git a/sdk/internal/zipstream/segment_writer.go b/sdk/internal/zipstream/segment_writer.go index 9f78d0ed89..7402fd9a1c 100644 --- a/sdk/internal/zipstream/segment_writer.go +++ b/sdk/internal/zipstream/segment_writer.go @@ -50,6 +50,9 @@ func NewSegmentTDFWriter(expectedSegments int, opts ...Option) SegmentWriter { // WriteSegment writes a segment with deterministic output based on segment index func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byte) ([]byte, error) { + // CRC32 over stored segment bytes (what goes into the ZIP entry) + originalCRC := crc32.ChecksumIEEE(data) + sw.mu.Lock() defer sw.mu.Unlock() @@ -79,12 +82,11 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt default: } - // CRC32 over stored segment bytes (what goes into the ZIP entry) - originalCRC := crc32.ChecksumIEEE(data) originalSize := uint64(len(data)) // Create segment buffer for this segment's output buffer := &bytes.Buffer{} + offset := 100 // Deterministic behavior: segment 0 gets ZIP header, others get raw data if index == 0 { @@ -92,6 +94,8 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt if err := sw.writeLocalFileHeader(buffer); err != nil { return nil, &Error{Op: "write-segment", Type: "segment", Err: err} } + offset = 100 - buffer.Len() + buffer.Write(data[offset:]) // Write remaining data after header } // All segments: write the encrypted data @@ -110,7 +114,7 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt sw.payloadEntry.CompressedSize += uint64(len(data)) // Encrypted size // Return the bytes for this segment - return buffer.Bytes(), nil + return data[offset:], nil } // Finalize completes the TDF creation with manifest and ZIP structures From 4130e206ef37a65a9addecbd24989bb29ded32fa Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 6 Nov 2025 21:55:18 -0800 Subject: [PATCH 02/17] no mem --- lib/ocrypto/aes_gcm.go | 13 ++++++------- sdk/experimental/tdf/writer.go | 17 ++++++++++++----- sdk/internal/zipstream/segment_writer.go | 9 +-------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/lib/ocrypto/aes_gcm.go b/lib/ocrypto/aes_gcm.go index 044be7f65b..6866587050 100644 --- a/lib/ocrypto/aes_gcm.go +++ b/lib/ocrypto/aes_gcm.go @@ -46,21 +46,20 @@ func (aesGcm AesGcm) Encrypt(data []byte) ([]byte, error) { cipherText := gcm.Seal(nonce, nonce, data, nil) return cipherText, nil } -func (aesGcm AesGcm) EncryptWithDestination(data []byte, dst []byte) ([]byte, error) { + +func (aesGcm AesGcm) EncryptInPlace(data []byte) ([]byte, []byte, error) { nonce, err := RandomBytes(GcmStandardNonceSize) if err != nil { - return nil, err + return nil, nil, err } - copy(dst[:GcmStandardNonceSize], nonce) - gcm, err := cipher.NewGCMWithNonceSize(aesGcm.block, GcmStandardNonceSize) if err != nil { - return nil, fmt.Errorf("cipher.NewGCMWithNonceSize failed: %w", err) + return nil, nil, fmt.Errorf("cipher.NewGCMWithNonceSize failed: %w", err) } - cipherText := gcm.Seal(dst[:GcmStandardNonceSize], nonce, data, nil) - return cipherText, nil + cipherText := gcm.Seal(data[:0], nonce, data, nil) + return cipherText, nonce, nil } // EncryptWithIV encrypts data with symmetric key. diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index 3174c01ba1..ec35b651dc 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" "sort" "sync" @@ -33,7 +34,8 @@ const ( // SegmentResult contains the result of writing a segment type SegmentResult struct { - Data []byte `json:"data"` // Encrypted segment bytes (for streaming) + Data []byte `json:"data"` // Encrypted segment bytes (for streaming) + TDFData io.Reader Index int `json:"index"` // Segment index Hash string `json:"hash"` // Base64-encoded integrity hash PlaintextSize int64 `json:"plaintextSize"` // Original data size @@ -253,13 +255,11 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg } w.mutex.Unlock() - segmentBlock := make([]byte, 100+len(data)+ 100) // Encrypt directly without unnecessary copying - the archive layer will handle copying if needed - segmentCipher, err := w.block.EncryptWithDestination(data, segmentBlock[:100]) + segmentCipher, nonce, err := w.block.EncryptInPlace(data) if err != nil { return nil, err } - segmentBlock = segmentBlock[:100+len(segmentCipher)] segmentSig, err := calculateSignature(segmentCipher, w.dek, w.segmentIntegrityAlgorithm, false) // Don't ever hex encode new tdf's if err != nil { return nil, err @@ -272,13 +272,20 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg EncryptedSize: int64(len(segmentCipher)), } - zipBytes, err := w.archiveWriter.WriteSegment(ctx, index, segmentBlock) + zipBytes, err := w.archiveWriter.WriteSegment(ctx, index, segmentCipher) if err != nil { return nil, err } + var reader io.Reader + if len(zipBytes) != 0 { + reader = io.MultiReader(bytes.NewReader(nonce), bytes.NewReader(segmentCipher), bytes.NewReader(zipBytes)) + } else { + reader = io.MultiReader(bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) + } return &SegmentResult{ Data: zipBytes, + TDFData: reader, Index: index, Hash: segmentHash, PlaintextSize: int64(len(data)), diff --git a/sdk/internal/zipstream/segment_writer.go b/sdk/internal/zipstream/segment_writer.go index 7402fd9a1c..80a8d75ab5 100644 --- a/sdk/internal/zipstream/segment_writer.go +++ b/sdk/internal/zipstream/segment_writer.go @@ -86,7 +86,6 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt // Create segment buffer for this segment's output buffer := &bytes.Buffer{} - offset := 100 // Deterministic behavior: segment 0 gets ZIP header, others get raw data if index == 0 { @@ -94,14 +93,8 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt if err := sw.writeLocalFileHeader(buffer); err != nil { return nil, &Error{Op: "write-segment", Type: "segment", Err: err} } - offset = 100 - buffer.Len() - buffer.Write(data[offset:]) // Write remaining data after header } - // All segments: write the encrypted data - if _, err := buffer.Write(data); err != nil { - return nil, &Error{Op: "write-segment", Type: "segment", Err: err} - } // Record segment metadata only (no payload retention). Payload bytes are returned // to the caller and may be uploaded; we keep only CRC and size for finalize. @@ -114,7 +107,7 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt sw.payloadEntry.CompressedSize += uint64(len(data)) // Encrypted size // Return the bytes for this segment - return data[offset:], nil + return buffer.Bytes(), nil } // Finalize completes the TDF creation with manifest and ZIP structures From 24ba5c61d687c13b724057a3f67e3252159fca74 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 6 Nov 2025 22:40:48 -0800 Subject: [PATCH 03/17] map write fix --- sdk/experimental/tdf/writer.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index ec35b651dc..68faed24b6 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -111,7 +111,7 @@ type Writer struct { // segments stores segment metadata using sparse map for memory efficiency // Maps segment index to Segment metadata (hash, size information) - segments map[int]Segment + segments map[int]*Segment // maxSegmentIndex tracks the highest segment index written maxSegmentIndex int @@ -184,7 +184,7 @@ func NewWriter(_ context.Context, opts ...Option[*WriterConfig]) (*Writer, error WriterConfig: *config, archiveWriter: archiveWriter, dek: dek, - segments: make(map[int]Segment), // Initialize sparse storage + segments: make(map[int]*Segment), // Initialize sparse storage block: block, initialAttributes: config.initialAttributes, initialDefaultKAS: config.initialDefaultKAS, @@ -253,6 +253,9 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg if index > w.maxSegmentIndex { w.maxSegmentIndex = index } + seg := &Segment{} + w.segments[index] = seg + w.mutex.Unlock() // Encrypt directly without unnecessary copying - the archive layer will handle copying if needed @@ -266,11 +269,9 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg } segmentHash := string(ocrypto.Base64Encode([]byte(segmentSig))) - w.segments[index] = Segment{ - Hash: segmentHash, - Size: int64(len(data)), // Use original data length - EncryptedSize: int64(len(segmentCipher)), - } + seg.Size = int64(len(data)) + seg.EncryptedSize = int64(len(segmentCipher)) + int64(len(nonce)) + seg.Hash = segmentHash zipBytes, err := w.archiveWriter.WriteSegment(ctx, index, segmentCipher) if err != nil { @@ -511,7 +512,7 @@ func (w *Writer) getManifest(ctx context.Context, cfg *WriterFinalizeConfig) (*M // Copy segments to manifest in finalize order (pack densely) for i, idx := range order { if segment, exists := w.segments[idx]; exists { - encryptInfo.Segments[i] = segment + encryptInfo.Segments[i] = *segment } } From 8117eebdb1cf05b0dc2342bd81f8ac64e9035de6 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Wed, 12 Nov 2025 12:23:29 -0800 Subject: [PATCH 04/17] fixes --- sdk/experimental/tdf/writer.go | 29 ++++++++++++++--------------- sdk/experimental/tdf/writer_test.go | 2 +- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index 68faed24b6..27f9d63c1c 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -34,13 +34,12 @@ const ( // SegmentResult contains the result of writing a segment type SegmentResult struct { - Data []byte `json:"data"` // Encrypted segment bytes (for streaming) - TDFData io.Reader - Index int `json:"index"` // Segment index - Hash string `json:"hash"` // Base64-encoded integrity hash - PlaintextSize int64 `json:"plaintextSize"` // Original data size - EncryptedSize int64 `json:"encryptedSize"` // Encrypted data size - CRC32 uint32 `json:"crc32"` // CRC32 checksum + TDFData io.Reader // Reader for the full TDF segment (nonce + encrypted data + zip structures) + Data []byte `json:"data"` // Encrypted segment bytes + Index int `json:"index"` // Segment index + Hash string `json:"hash"` // Base64-encoded integrity hash + PlaintextSize int64 `json:"plaintextSize"` // Original data size + EncryptedSize int64 `json:"encryptedSize"` // Encrypted data size } // FinalizeResult contains the complete TDF creation result @@ -273,24 +272,24 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg seg.EncryptedSize = int64(len(segmentCipher)) + int64(len(nonce)) seg.Hash = segmentHash - zipBytes, err := w.archiveWriter.WriteSegment(ctx, index, segmentCipher) + header, err := w.archiveWriter.WriteSegment(ctx, index, segmentCipher) if err != nil { return nil, err } var reader io.Reader - if len(zipBytes) != 0 { - reader = io.MultiReader(bytes.NewReader(nonce), bytes.NewReader(segmentCipher), bytes.NewReader(zipBytes)) - } else { + if len(header) != 0 { reader = io.MultiReader(bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) + } else { + reader = io.MultiReader(bytes.NewReader(header), bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) } return &SegmentResult{ - Data: zipBytes, TDFData: reader, + Data: segmentCipher, Index: index, - Hash: segmentHash, - PlaintextSize: int64(len(data)), - EncryptedSize: int64(len(zipBytes)), + Hash: seg.Hash, + PlaintextSize: seg.Size, + EncryptedSize: seg.EncryptedSize, }, nil } diff --git a/sdk/experimental/tdf/writer_test.go b/sdk/experimental/tdf/writer_test.go index a4d8251af5..296c582041 100644 --- a/sdk/experimental/tdf/writer_test.go +++ b/sdk/experimental/tdf/writer_test.go @@ -669,7 +669,7 @@ func testErrorConditions(t *testing.T) { require.NoError(t, err) // Manually corrupt segment hash to test error handling - writer.segments[0] = Segment{Hash: "", Size: 10, EncryptedSize: 26} + writer.segments[0] = &Segment{Hash: "", Size: 10, EncryptedSize: 26} writer.maxSegmentIndex = 0 attributes := []*policy.Value{ From 3d554481b1d9f1ca4963477b7819ed2e1db7df78 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Wed, 12 Nov 2025 13:27:03 -0800 Subject: [PATCH 05/17] bug fixes --- sdk/experimental/tdf/writer.go | 2 +- sdk/internal/zipstream/segment_writer.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index 27f9d63c1c..987c9ebf01 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -277,7 +277,7 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg return nil, err } var reader io.Reader - if len(header) != 0 { + if len(header) == 0 { reader = io.MultiReader(bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) } else { reader = io.MultiReader(bytes.NewReader(header), bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) diff --git a/sdk/internal/zipstream/segment_writer.go b/sdk/internal/zipstream/segment_writer.go index 80a8d75ab5..7a2f86c5b8 100644 --- a/sdk/internal/zipstream/segment_writer.go +++ b/sdk/internal/zipstream/segment_writer.go @@ -11,6 +11,8 @@ import ( "sort" "sync" "time" + + "github.com/opentdf/platform/lib/ocrypto" ) // segmentWriter implements the SegmentWriter interface for out-of-order segment writing @@ -82,7 +84,7 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt default: } - originalSize := uint64(len(data)) + originalSize := uint64(len(data)) + ocrypto.GcmStandardNonceSize // Create segment buffer for this segment's output buffer := &bytes.Buffer{} @@ -104,7 +106,7 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt // Update payload entry metadata sw.payloadEntry.Size += originalSize - sw.payloadEntry.CompressedSize += uint64(len(data)) // Encrypted size + sw.payloadEntry.CompressedSize += originalSize // Encrypted size // Return the bytes for this segment return buffer.Bytes(), nil From 58613462b66cfdbd73a295a29300c10b076f141b Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Wed, 12 Nov 2025 13:27:10 -0800 Subject: [PATCH 06/17] benchmark --- examples/cmd/benchmark_experimental.go | 101 +++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 examples/cmd/benchmark_experimental.go diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go new file mode 100644 index 0000000000..28bed81017 --- /dev/null +++ b/examples/cmd/benchmark_experimental.go @@ -0,0 +1,101 @@ +//nolint:forbidigo // We use Println here extensively because we are printing markdown. +package cmd + +import ( + "context" + "fmt" + "math/rand" + "sync" + "time" + + "connectrpc.com/connect" + "github.com/opentdf/platform/lib/ocrypto" + kasp "github.com/opentdf/platform/protocol/go/kas" + "github.com/opentdf/platform/protocol/go/kas/kasconnect" + "github.com/opentdf/platform/protocol/go/policy" + + "github.com/opentdf/platform/sdk/experimental/tdf" + "github.com/opentdf/platform/sdk/httputil" + "github.com/spf13/cobra" +) + +var payloadSize int + +func init() { + benchmarkCmd := &cobra.Command{ + Use: "benchmark-experimental-writer", + Short: "Benchmark experimental TDF writer speed", + Long: `Benchmark the experimental TDF writer with configurable payload size.`, + RunE: runExperimentalWriterBenchmark, + } + benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB + ExamplesCmd.AddCommand(benchmarkCmd) +} + +func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { + payload := make([]byte, payloadSize) + rand.Read(payload) + + http := httputil.SafeHTTPClient() + fmt.Println("endpoint:", platformEndpoint) + serviceClient := kasconnect.NewAccessServiceClient(http, platformEndpoint) + resp, err := serviceClient.PublicKey(context.Background(), connect.NewRequest(&kasp.PublicKeyRequest{Algorithm: string(ocrypto.RSA2048Key)})) + if err != nil { + return fmt.Errorf("failed to get public key from KAS: %w", err) + } + attr := "https://example.com/attr/attr1/value/value1" + var attrs []*policy.Value + + simpleyKey := &policy.SimpleKasKey{ + KasUri: platformEndpoint, + KasId: "id", + PublicKey: &policy.SimpleKasPublicKey{ + Kid: resp.Msg.Kid, + Pem: resp.Msg.PublicKey, + Algorithm: policy.Algorithm_ALGORITHM_RSA_2048, + }, + } + + attrs = append(attrs, &policy.Value{Fqn: attr, KasKeys: []*policy.SimpleKasKey{simpleyKey}, Attribute: &policy.Attribute{Namespace: &policy.Namespace{Name: "example.com"}, Fqn: attr}}) + writer, err := tdf.NewWriter(context.Background(), tdf.WithDefaultKASForWriter(simpleyKey), tdf.WithInitialAttributes(attrs), tdf.WithSegmentIntegrityAlgorithm(tdf.HS256)) + if err != nil { + return fmt.Errorf("failed to create writer: %w", err) + } + segmentChunk := 64 * 1024 // 64KB chunks + i := 0 + wg := sync.WaitGroup{} + segs := len(payload) / segmentChunk + wg.Add(segs) + start := time.Now() + for i < segs { + segment := i + func() { + start := i * segmentChunk + end := min(start+segmentChunk, len(payload)) + _, err = writer.WriteSegment(context.Background(), segment, payload[start:end]) + if err != nil { + fmt.Println(err) + panic(err) + } + wg.Done() + }() + i++ + } + wg.Wait() + + end := time.Now() + result, err := writer.Finalize(context.Background()) + if err != nil { + return fmt.Errorf("failed to finalize writer: %w", err) + } + totalTime := end.Sub(start) + + fmt.Printf("# Benchmark Experimental TDF Writer Results:\n") + fmt.Printf("| Metric | Value |\n") + fmt.Printf("|--------------------|--------------|\n") + fmt.Printf("| Payload Size (B) | %d |\n", payloadSize) + fmt.Printf("| Output Size (B) | %d |\n", len(result.Data)) + fmt.Printf("| Total Time | %s |\n", totalTime) + + return nil +} From d633c440e34baa13dfa033ff1d7f7c179aa2933f Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Wed, 12 Nov 2025 16:04:22 -0800 Subject: [PATCH 07/17] lint fixes --- examples/cmd/benchmark_experimental.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go index 28bed81017..2ae73d968d 100644 --- a/examples/cmd/benchmark_experimental.go +++ b/examples/cmd/benchmark_experimental.go @@ -3,8 +3,8 @@ package cmd import ( "context" + "crypto/rand" "fmt" - "math/rand" "sync" "time" @@ -19,7 +19,10 @@ import ( "github.com/spf13/cobra" ) -var payloadSize int +var ( + payloadSize int + segmentChunk int +) func init() { benchmarkCmd := &cobra.Command{ @@ -28,13 +31,17 @@ func init() { Long: `Benchmark the experimental TDF writer with configurable payload size.`, RunE: runExperimentalWriterBenchmark, } - benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB + benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB //nolint + benchmarkCmd.Flags().IntVar(&segmentChunk, "segment-chunks", 16*1024, "segment chunks ize") // Default 16 segments //nolint ExamplesCmd.AddCommand(benchmarkCmd) } func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { payload := make([]byte, payloadSize) - rand.Read(payload) + _, err := rand.Read(payload) + if err != nil { + return fmt.Errorf("failed to generate random payload: %w", err) + } http := httputil.SafeHTTPClient() fmt.Println("endpoint:", platformEndpoint) @@ -43,7 +50,6 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("failed to get public key from KAS: %w", err) } - attr := "https://example.com/attr/attr1/value/value1" var attrs []*policy.Value simpleyKey := &policy.SimpleKasKey{ @@ -61,7 +67,6 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("failed to create writer: %w", err) } - segmentChunk := 64 * 1024 // 64KB chunks i := 0 wg := sync.WaitGroup{} segs := len(payload) / segmentChunk From 7cb700cb381324bf3811fd2965974bf2a5e34b4e Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Wed, 12 Nov 2025 16:14:54 -0800 Subject: [PATCH 08/17] lint fixes --- examples/cmd/benchmark_experimental.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go index 2ae73d968d..4c33fab613 100644 --- a/examples/cmd/benchmark_experimental.go +++ b/examples/cmd/benchmark_experimental.go @@ -31,8 +31,10 @@ func init() { Long: `Benchmark the experimental TDF writer with configurable payload size.`, RunE: runExperimentalWriterBenchmark, } - benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB //nolint - benchmarkCmd.Flags().IntVar(&segmentChunk, "segment-chunks", 16*1024, "segment chunks ize") // Default 16 segments //nolint + //nolint: mnd + benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB + //nolint: mnd + benchmarkCmd.Flags().IntVar(&segmentChunk, "segment-chunks", 16*1024, "segment chunks ize") // Default 16 segments ExamplesCmd.AddCommand(benchmarkCmd) } @@ -56,8 +58,8 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { KasUri: platformEndpoint, KasId: "id", PublicKey: &policy.SimpleKasPublicKey{ - Kid: resp.Msg.Kid, - Pem: resp.Msg.PublicKey, + Kid: resp.Msg.GetKid(), + Pem: resp.Msg.GetPublicKey(), Algorithm: policy.Algorithm_ALGORITHM_RSA_2048, }, } From 0cb2032c6aff55a94fabe9866acba119b60b7bb7 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Wed, 12 Nov 2025 16:18:27 -0800 Subject: [PATCH 09/17] fomat file --- examples/cmd/benchmark_experimental.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go index 4c33fab613..25c147ac64 100644 --- a/examples/cmd/benchmark_experimental.go +++ b/examples/cmd/benchmark_experimental.go @@ -31,9 +31,9 @@ func init() { Long: `Benchmark the experimental TDF writer with configurable payload size.`, RunE: runExperimentalWriterBenchmark, } - //nolint: mnd + //nolint: mnd benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB - //nolint: mnd + //nolint: mnd benchmarkCmd.Flags().IntVar(&segmentChunk, "segment-chunks", 16*1024, "segment chunks ize") // Default 16 segments ExamplesCmd.AddCommand(benchmarkCmd) } From 274be83c15f5d0762e31012cbc099a9ddf98ce08 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Wed, 12 Nov 2025 16:23:02 -0800 Subject: [PATCH 10/17] lint fixes --- examples/cmd/benchmark_experimental.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go index 25c147ac64..39011f7719 100644 --- a/examples/cmd/benchmark_experimental.go +++ b/examples/cmd/benchmark_experimental.go @@ -31,9 +31,9 @@ func init() { Long: `Benchmark the experimental TDF writer with configurable payload size.`, RunE: runExperimentalWriterBenchmark, } - //nolint: mnd + //nolint: mnd // no magic number, this is just default value for payload size benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB - //nolint: mnd + //nolint: mnd // same as above benchmarkCmd.Flags().IntVar(&segmentChunk, "segment-chunks", 16*1024, "segment chunks ize") // Default 16 segments ExamplesCmd.AddCommand(benchmarkCmd) } From 157ab532dc8e4c3f0f3c589eb117cd0cd095fcff Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Wed, 12 Nov 2025 16:33:14 -0800 Subject: [PATCH 11/17] fix go mod --- examples/go.mod | 4 ++-- examples/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/go.mod b/examples/go.mod index dd48e2eaef..0adaedfae2 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -5,9 +5,10 @@ go 1.24.0 toolchain go1.24.9 require ( + connectrpc.com/connect v1.18.1 github.com/opentdf/platform/lib/ocrypto v0.7.0 github.com/opentdf/platform/protocol/go v0.13.0 - github.com/opentdf/platform/sdk v0.7.0 + github.com/opentdf/platform/sdk v0.10.1 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 google.golang.org/grpc v1.73.0 @@ -16,7 +17,6 @@ require ( require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250603165357-b52ab10f4468.1 // indirect - connectrpc.com/connect v1.18.1 // indirect github.com/Masterminds/semver/v3 v3.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/examples/go.sum b/examples/go.sum index 3b4cd41af2..b05ea7e203 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -120,8 +120,8 @@ github.com/opentdf/platform/lib/ocrypto v0.7.0 h1:uBZJXisuXU3V8681aP8FVMJkyWBrwW github.com/opentdf/platform/lib/ocrypto v0.7.0/go.mod h1:sYhoBL1bQYgQVSSNpxU13RsrE5JAk8BABT1hfr9L3j8= github.com/opentdf/platform/protocol/go v0.13.0 h1:vrOOHyhYDPzJgNenz/1g0M5nWtkOYKkPggMNHKzeMcs= github.com/opentdf/platform/protocol/go v0.13.0/go.mod h1:GRycoDGDxaz91sOvGZFWVEKJLluZFg2wM3NJmhucDHo= -github.com/opentdf/platform/sdk v0.7.0 h1:8hczDycXGY1ucdIXSrP17oW/Eyu3vsb4LEX4hc7tvVY= -github.com/opentdf/platform/sdk v0.7.0/go.mod h1:CTJR1NXeYe896M1/VN0h+1Ff54SdBtxv4z18BGTi8yk= +github.com/opentdf/platform/sdk v0.10.1 h1:kBrTK48xle7mdGc+atlr4kDh94f6kVj+0OB76K8rozI= +github.com/opentdf/platform/sdk v0.10.1/go.mod h1:+yaTi/c/GWHZPPmO27sq2s7Tcb2P/USkK8LuW1krhI8= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 2fb31ce70e3cf3f89a51e4d6b2cb5b08b2d46e5e Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 13 Nov 2025 11:36:52 -0800 Subject: [PATCH 12/17] test fixes --- examples/cmd/benchmark_experimental.go | 108 ------------------ sdk/experimental/tdf/writer.go | 12 +- sdk/internal/zipstream/benchmark_test.go | 16 ++- sdk/internal/zipstream/segment_writer.go | 12 +- sdk/internal/zipstream/segment_writer_test.go | 69 ++++++----- sdk/internal/zipstream/writer.go | 2 +- sdk/internal/zipstream/zip64_mode_test.go | 13 ++- sdk/internal/zipstream/zip_primitives.go | 2 +- 8 files changed, 74 insertions(+), 160 deletions(-) delete mode 100644 examples/cmd/benchmark_experimental.go diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go deleted file mode 100644 index 39011f7719..0000000000 --- a/examples/cmd/benchmark_experimental.go +++ /dev/null @@ -1,108 +0,0 @@ -//nolint:forbidigo // We use Println here extensively because we are printing markdown. -package cmd - -import ( - "context" - "crypto/rand" - "fmt" - "sync" - "time" - - "connectrpc.com/connect" - "github.com/opentdf/platform/lib/ocrypto" - kasp "github.com/opentdf/platform/protocol/go/kas" - "github.com/opentdf/platform/protocol/go/kas/kasconnect" - "github.com/opentdf/platform/protocol/go/policy" - - "github.com/opentdf/platform/sdk/experimental/tdf" - "github.com/opentdf/platform/sdk/httputil" - "github.com/spf13/cobra" -) - -var ( - payloadSize int - segmentChunk int -) - -func init() { - benchmarkCmd := &cobra.Command{ - Use: "benchmark-experimental-writer", - Short: "Benchmark experimental TDF writer speed", - Long: `Benchmark the experimental TDF writer with configurable payload size.`, - RunE: runExperimentalWriterBenchmark, - } - //nolint: mnd // no magic number, this is just default value for payload size - benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB - //nolint: mnd // same as above - benchmarkCmd.Flags().IntVar(&segmentChunk, "segment-chunks", 16*1024, "segment chunks ize") // Default 16 segments - ExamplesCmd.AddCommand(benchmarkCmd) -} - -func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { - payload := make([]byte, payloadSize) - _, err := rand.Read(payload) - if err != nil { - return fmt.Errorf("failed to generate random payload: %w", err) - } - - http := httputil.SafeHTTPClient() - fmt.Println("endpoint:", platformEndpoint) - serviceClient := kasconnect.NewAccessServiceClient(http, platformEndpoint) - resp, err := serviceClient.PublicKey(context.Background(), connect.NewRequest(&kasp.PublicKeyRequest{Algorithm: string(ocrypto.RSA2048Key)})) - if err != nil { - return fmt.Errorf("failed to get public key from KAS: %w", err) - } - var attrs []*policy.Value - - simpleyKey := &policy.SimpleKasKey{ - KasUri: platformEndpoint, - KasId: "id", - PublicKey: &policy.SimpleKasPublicKey{ - Kid: resp.Msg.GetKid(), - Pem: resp.Msg.GetPublicKey(), - Algorithm: policy.Algorithm_ALGORITHM_RSA_2048, - }, - } - - attrs = append(attrs, &policy.Value{Fqn: attr, KasKeys: []*policy.SimpleKasKey{simpleyKey}, Attribute: &policy.Attribute{Namespace: &policy.Namespace{Name: "example.com"}, Fqn: attr}}) - writer, err := tdf.NewWriter(context.Background(), tdf.WithDefaultKASForWriter(simpleyKey), tdf.WithInitialAttributes(attrs), tdf.WithSegmentIntegrityAlgorithm(tdf.HS256)) - if err != nil { - return fmt.Errorf("failed to create writer: %w", err) - } - i := 0 - wg := sync.WaitGroup{} - segs := len(payload) / segmentChunk - wg.Add(segs) - start := time.Now() - for i < segs { - segment := i - func() { - start := i * segmentChunk - end := min(start+segmentChunk, len(payload)) - _, err = writer.WriteSegment(context.Background(), segment, payload[start:end]) - if err != nil { - fmt.Println(err) - panic(err) - } - wg.Done() - }() - i++ - } - wg.Wait() - - end := time.Now() - result, err := writer.Finalize(context.Background()) - if err != nil { - return fmt.Errorf("failed to finalize writer: %w", err) - } - totalTime := end.Sub(start) - - fmt.Printf("# Benchmark Experimental TDF Writer Results:\n") - fmt.Printf("| Metric | Value |\n") - fmt.Printf("|--------------------|--------------|\n") - fmt.Printf("| Payload Size (B) | %d |\n", payloadSize) - fmt.Printf("| Output Size (B) | %d |\n", len(result.Data)) - fmt.Printf("| Total Time | %s |\n", totalTime) - - return nil -} diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index 987c9ebf01..76bbe6c8ac 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "hash/crc32" "io" "log/slog" "sort" @@ -272,7 +273,16 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg seg.EncryptedSize = int64(len(segmentCipher)) + int64(len(nonce)) seg.Hash = segmentHash - header, err := w.archiveWriter.WriteSegment(ctx, index, segmentCipher) + crc := crc32.NewIEEE() + _, err = crc.Write(nonce) + if err != nil { + return nil, err + } + _, err = crc.Write(segmentCipher) + if err != nil { + return nil, err + } + header, err := w.archiveWriter.WriteSegment(ctx, index, uint64(seg.EncryptedSize), crc.Sum32()) if err != nil { return nil, err } diff --git a/sdk/internal/zipstream/benchmark_test.go b/sdk/internal/zipstream/benchmark_test.go index 4ef7e38d83..24263c1d3d 100644 --- a/sdk/internal/zipstream/benchmark_test.go +++ b/sdk/internal/zipstream/benchmark_test.go @@ -3,6 +3,7 @@ package zipstream import ( + "hash/crc32" "testing" ) @@ -55,7 +56,8 @@ func BenchmarkSegmentWriter_CRC32ContiguousProcessing(b *testing.B) { // Write segments in specified order for _, segIdx := range writeOrder { - _, err := writer.WriteSegment(ctx, segIdx, segmentData) + crc := crc32.ChecksumIEEE(segmentData) + _, err := writer.WriteSegment(ctx, segIdx, uint64(len(segmentData)), crc) if err != nil { b.Fatal(err) } @@ -99,7 +101,8 @@ func BenchmarkSegmentWriter_SparseIndices(b *testing.B) { // Write sparse indices in order for k := 0; k < n; k++ { idx := k * stride - if _, err := w.WriteSegment(ctx, idx, data); err != nil { + crc := crc32.ChecksumIEEE(data) + if _, err := w.WriteSegment(ctx, idx, uint64(len(data)), crc); err != nil { b.Fatal(err) } } @@ -153,7 +156,8 @@ func BenchmarkSegmentWriter_VariableSegmentSizes(b *testing.B) { segmentData[j] = byte((segIdx * j) % 256) } - _, err := writer.WriteSegment(ctx, segIdx, segmentData) + crc := crc32.ChecksumIEEE(segmentData) + _, err := writer.WriteSegment(ctx, segIdx, uint64(len(segmentData)), crc) if err != nil { b.Fatal(err) } @@ -234,7 +238,8 @@ func BenchmarkSegmentWriter_MemoryPressure(b *testing.B) { segmentData[j] = byte((orderIdx * j) % 256) } - _, err := writer.WriteSegment(ctx, segIdx, segmentData) + crc := crc32.ChecksumIEEE(segmentData) + _, err := writer.WriteSegment(ctx, segIdx, uint64(len(segmentData)), crc) if err != nil { b.Fatal(err) } @@ -305,7 +310,8 @@ func BenchmarkSegmentWriter_ZIPGeneration(b *testing.B) { // Write all segments for segIdx := 0; segIdx < tc.segmentCount; segIdx++ { - _, err := writer.WriteSegment(ctx, segIdx, segmentData) + crc := crc32.ChecksumIEEE(segmentData) + _, err := writer.WriteSegment(ctx, segIdx, uint64(len(segmentData)), crc) if err != nil { b.Fatal(err) } diff --git a/sdk/internal/zipstream/segment_writer.go b/sdk/internal/zipstream/segment_writer.go index 7a2f86c5b8..2c2500be8d 100644 --- a/sdk/internal/zipstream/segment_writer.go +++ b/sdk/internal/zipstream/segment_writer.go @@ -11,8 +11,6 @@ import ( "sort" "sync" "time" - - "github.com/opentdf/platform/lib/ocrypto" ) // segmentWriter implements the SegmentWriter interface for out-of-order segment writing @@ -51,10 +49,7 @@ func NewSegmentTDFWriter(expectedSegments int, opts ...Option) SegmentWriter { } // WriteSegment writes a segment with deterministic output based on segment index -func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byte) ([]byte, error) { - // CRC32 over stored segment bytes (what goes into the ZIP entry) - originalCRC := crc32.ChecksumIEEE(data) - +func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, size uint64, crc32 uint32) ([]byte, error) { sw.mu.Lock() defer sw.mu.Unlock() @@ -84,7 +79,7 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt default: } - originalSize := uint64(len(data)) + ocrypto.GcmStandardNonceSize + originalSize := size // Create segment buffer for this segment's output buffer := &bytes.Buffer{} @@ -97,10 +92,9 @@ func (sw *segmentWriter) WriteSegment(ctx context.Context, index int, data []byt } } - // Record segment metadata only (no payload retention). Payload bytes are returned // to the caller and may be uploaded; we keep only CRC and size for finalize. - if err := sw.metadata.AddSegment(index, data, originalSize, originalCRC); err != nil { + if err := sw.metadata.AddSegment(index, size, crc32); err != nil { return nil, &Error{Op: "write-segment", Type: "segment", Err: err} } diff --git a/sdk/internal/zipstream/segment_writer_test.go b/sdk/internal/zipstream/segment_writer_test.go index 130af5aa21..b00ff273e6 100644 --- a/sdk/internal/zipstream/segment_writer_test.go +++ b/sdk/internal/zipstream/segment_writer_test.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "fmt" + "hash/crc32" "io" "testing" @@ -32,21 +33,22 @@ func TestSegmentWriter_SequentialOrder(t *testing.T) { // Write segments in sequential order for i, data := range testSegments { - segmentBytes, err := writer.WriteSegment(ctx, i, data) + crc := crc32.ChecksumIEEE(data) + segmentBytes, err := writer.WriteSegment(ctx, i, uint64(len(data)), crc) require.NoError(t, err, "Failed to write segment %d", i) - assert.NotEmpty(t, segmentBytes, "Segment %d should have bytes", i) t.Logf("Sequential segment %d: %d bytes", i, len(segmentBytes)) if i == 0 { // Segment 0 should be larger due to ZIP header - assert.Greater(t, len(segmentBytes), len(data), "Segment 0 should include ZIP header") + assert.NotEmpty(t, segmentBytes, "Segment 0 should include ZIP header") } else { // Other segments should be approximately the size of the data - assert.Len(t, data, len(segmentBytes), "Segment %d should be raw data", i) + assert.Empty(t, segmentBytes, "Segment %d should have no zip bytes", i) } allBytes = append(allBytes, segmentBytes...) + allBytes = append(allBytes, data...) } t.Logf("Sequential total payload bytes before finalization: %d", len(allBytes)) @@ -102,16 +104,19 @@ func TestSegmentWriter_OutOfOrder(t *testing.T) { for _, index := range writeOrder { data := testSegments[index] - bytes, err := writer.WriteSegment(ctx, index, data) + crc := crc32.ChecksumIEEE(data) + bytes, err := writer.WriteSegment(ctx, index, uint64(len(data)), crc) require.NoError(t, err, "Failed to write segment %d out of order", index) - assert.NotEmpty(t, bytes, "Segment %d should have bytes", index) if index == 0 { // Segment 0 should always include ZIP header, regardless of write order - assert.Greater(t, len(bytes), len(data), "Segment 0 should include ZIP header even when written out of order") + assert.NotEmpty(t, bytes, "Segment 0 should include ZIP header") + } else { + assert.Empty(t, bytes, "Segment %d should have no zip bytes", index) } - segmentBytes[index] = bytes + allBytes := append(bytes, data...) + segmentBytes[index] = allBytes } // Reassemble in logical order (as S3 would do) @@ -180,15 +185,16 @@ func TestSegmentWriter_SparseIndices_InOrder(t *testing.T) { segmentBytes := make(map[int][]byte) for _, index := range order { data := testSegments[index] - bytes, err := writer.WriteSegment(ctx, index, data) + crc := crc32.ChecksumIEEE(data) + bytes, err := writer.WriteSegment(ctx, index, uint64(len(data)), crc) require.NoError(t, err, "write segment %d failed", index) - assert.NotEmpty(t, bytes, "segment %d should yield bytes", index) if index == 0 { - assert.Greater(t, len(bytes), len(data), "segment 0 should include ZIP header") + assert.NotEmpty(t, bytes, "segment 0 should include ZIP header") } else { - assert.Len(t, bytes, len(data), "non-zero segments are raw payload bytes") + assert.Empty(t, bytes, "segment %d should have no zip bytes", index) } - segmentBytes[index] = bytes + totalBytes := append(bytes, data...) + segmentBytes[index] = totalBytes } // Assemble full file: concatenate segment bytes in ascending index order @@ -244,10 +250,11 @@ func TestSegmentWriter_SparseIndices_OutOfOrder(t *testing.T) { segmentBytes := make(map[int][]byte) for _, index := range writeOrder { data := testSegments[index] - bytes, err := writer.WriteSegment(ctx, index, data) + crc := crc32.ChecksumIEEE(data) + bytes, err := writer.WriteSegment(ctx, index, uint64(len(data)), crc) require.NoError(t, err, "write segment %d failed", index) - assert.NotEmpty(t, bytes, "segment %d should yield bytes", index) - segmentBytes[index] = bytes + totalBytes := append(bytes, data...) + segmentBytes[index] = totalBytes } // Assemble full file in final (ascending) order regardless of write order @@ -285,10 +292,10 @@ func TestSegmentWriter_DuplicateSegments(t *testing.T) { ctx := t.Context() // Write segment 1 twice - _, err := writer.WriteSegment(ctx, 1, []byte("first")) + _, err := writer.WriteSegment(ctx, 1, 10, crc32.ChecksumIEEE([]byte("first write"))) require.NoError(t, err, "First write of segment 1 should succeed") - _, err = writer.WriteSegment(ctx, 1, []byte("duplicate")) + _, err = writer.WriteSegment(ctx, 1, 10, crc32.ChecksumIEEE([]byte("second write"))) require.Error(t, err, "Duplicate segment should fail") assert.Contains(t, err.Error(), "duplicate", "Error should mention duplicate") @@ -310,7 +317,7 @@ func TestSegmentWriter_InvalidSegmentIndex(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := writer.WriteSegment(ctx, tc.index, []byte("test")) + _, err := writer.WriteSegment(ctx, tc.index, 10, crc32.ChecksumIEEE([]byte("test"))) require.Error(t, err, "Negative segment index should fail") assert.Contains(t, err.Error(), "invalid", "Error should mention invalid") }) @@ -318,7 +325,7 @@ func TestSegmentWriter_InvalidSegmentIndex(t *testing.T) { // Test that large indices are actually allowed (dynamic expansion) t.Run("large_index_allowed", func(t *testing.T) { - _, err := writer.WriteSegment(ctx, 100, []byte("test")) + _, err := writer.WriteSegment(ctx, 100, 10, crc32.ChecksumIEEE([]byte("large index"))) require.NoError(t, err, "Large segment index should be allowed for dynamic expansion") }) @@ -331,13 +338,13 @@ func TestSegmentWriter_AllowsGapsOnFinalize(t *testing.T) { ctx := t.Context() // Write only segments 0, 1, 3 (2 is missing) - _, err := writer.WriteSegment(ctx, 0, []byte("first")) + _, err := writer.WriteSegment(ctx, 0, 5, crc32.ChecksumIEEE([]byte("first"))) require.NoError(t, err) - _, err = writer.WriteSegment(ctx, 1, []byte("second")) + _, err = writer.WriteSegment(ctx, 1, 6, crc32.ChecksumIEEE([]byte("second"))) require.NoError(t, err) - _, err = writer.WriteSegment(ctx, 3, []byte("fourth")) + _, err = writer.WriteSegment(ctx, 3, 5, crc32.ChecksumIEEE([]byte("fourth"))) require.NoError(t, err) // Finalize should succeed (auto-dense behavior) @@ -356,7 +363,7 @@ func TestSegmentWriter_CleanupSegment(t *testing.T) { testData := []byte("test data for cleanup") // Write a segment - _, err := writer.WriteSegment(ctx, 1, testData) + _, err := writer.WriteSegment(ctx, 1, uint64(len(testData)), crc32.ChecksumIEEE(testData)) require.NoError(t, err) // Verify segment exists before cleanup @@ -385,7 +392,7 @@ func TestSegmentWriter_ContextCancellation(t *testing.T) { cancel() // Try to write segment with cancelled context - _, err := writer.WriteSegment(ctx, 0, []byte("test")) + _, err := writer.WriteSegment(ctx, 0, 10, crc32.ChecksumIEEE([]byte("data"))) require.Error(t, err, "Should fail with cancelled context") assert.Contains(t, err.Error(), "context", "Error should mention context") @@ -408,7 +415,7 @@ func TestSegmentWriter_LargeNumberOfSegments(t *testing.T) { // Write all segments in reverse order for i := segmentCount - 1; i >= 0; i-- { - bytes, err := writer.WriteSegment(ctx, i, testSegments[i]) + bytes, err := writer.WriteSegment(ctx, i, uint64(len(testSegments[i])), crc32.ChecksumIEEE(testSegments[i])) require.NoError(t, err, "Failed to write segment %d", i) // Store in logical order for final assembly @@ -439,13 +446,13 @@ func TestSegmentWriter_EmptySegments(t *testing.T) { ctx := t.Context() // Write segments with empty data - _, err := writer.WriteSegment(ctx, 0, []byte("")) + _, err := writer.WriteSegment(ctx, 0, 0, 0) require.NoError(t, err, "Should handle empty segment 0") - _, err = writer.WriteSegment(ctx, 1, []byte("non-empty")) + _, err = writer.WriteSegment(ctx, 1, 10, crc32.ChecksumIEEE([]byte("not empty"))) require.NoError(t, err, "Should handle non-empty segment") - _, err = writer.WriteSegment(ctx, 2, []byte("")) + _, err = writer.WriteSegment(ctx, 2, 0, 0) require.NoError(t, err, "Should handle empty segment 2") // Finalize @@ -500,7 +507,9 @@ func benchmarkSegmentWriter(b *testing.B, name string, writeOrder []int) { // Write segments in specified order for _, index := range writeOrder { - _, err := writer.WriteSegment(ctx, index, testSegments[index]) + data := testSegments[index] + crc := crc32.ChecksumIEEE(data) + _, err := writer.WriteSegment(ctx, index, uint64(len(data)), crc) if err != nil { b.Fatal(err) } diff --git a/sdk/internal/zipstream/writer.go b/sdk/internal/zipstream/writer.go index 6a2f5c8106..bebd50fcc8 100644 --- a/sdk/internal/zipstream/writer.go +++ b/sdk/internal/zipstream/writer.go @@ -22,7 +22,7 @@ type Writer interface { // SegmentWriter handles out-of-order segments with deterministic output type SegmentWriter interface { Writer - WriteSegment(ctx context.Context, index int, data []byte) ([]byte, error) + WriteSegment(ctx context.Context, index int, size uint64, crc32 uint32) ([]byte, error) Finalize(ctx context.Context, manifest []byte) ([]byte, error) // CleanupSegment removes the presence marker for a segment index. // Calling this before Finalize will cause IsComplete() to fail for that index. diff --git a/sdk/internal/zipstream/zip64_mode_test.go b/sdk/internal/zipstream/zip64_mode_test.go index 8c57613a50..b986c35846 100644 --- a/sdk/internal/zipstream/zip64_mode_test.go +++ b/sdk/internal/zipstream/zip64_mode_test.go @@ -5,6 +5,7 @@ package zipstream import ( "archive/zip" "bytes" + "hash/crc32" "io" "testing" ) @@ -24,16 +25,18 @@ func TestZip64Mode_Auto_Small_UsesZip32(t *testing.T) { w := NewSegmentTDFWriter(2, WithZip64Mode(Zip64Auto)) var parts [][]byte - p0, err := w.WriteSegment(t.Context(), 0, []byte("hello ")) + p0, err := w.WriteSegment(t.Context(), 0, 5, crc32.ChecksumIEEE([]byte("hello"))) if err != nil { t.Fatal(err) } parts = append(parts, p0) - p1, err := w.WriteSegment(t.Context(), 1, []byte("world")) + parts = append(parts, []byte("hello")) + p1, err := w.WriteSegment(t.Context(), 1, 5, crc32.ChecksumIEEE([]byte("world"))) if err != nil { t.Fatal(err) } parts = append(parts, p1) + parts = append(parts, []byte("world")) fin, err := w.Finalize(t.Context(), []byte(`{"m":1}`)) if err != nil { @@ -75,7 +78,7 @@ func TestZip64Mode_Auto_Small_UsesZip32(t *testing.T) { func TestZip64Mode_Always_Small_UsesZip64(t *testing.T) { w := NewSegmentTDFWriter(1, WithZip64Mode(Zip64Always)) - seg, err := w.WriteSegment(t.Context(), 0, []byte("data")) + seg, err := w.WriteSegment(t.Context(), 0, 4, crc32.ChecksumIEEE([]byte("data"))) if err != nil { t.Fatal(err) } @@ -85,7 +88,7 @@ func TestZip64Mode_Always_Small_UsesZip64(t *testing.T) { } w.Close() - data := buildZip(t, [][]byte{seg}, fin) + data := buildZip(t, [][]byte{seg, []byte("data")}, fin) // Basic open check; many readers accept ZIP64 regardless of size. if _, err := zip.NewReader(bytes.NewReader(data), int64(len(data))); err != nil { t.Fatalf("zip open failed (zip64 always): %v", err) @@ -101,7 +104,7 @@ func TestZip64Mode_Never_Overflow_Fails(t *testing.T) { t.Fatal("writer type assertion failed") } // Write minimal segment to initialize structures - if _, err := w.WriteSegment(t.Context(), 0, []byte("x")); err != nil { + if _, err := w.WriteSegment(t.Context(), 0, 1, crc32.ChecksumIEEE([]byte("x"))); err != nil { t.Fatal(err) } sw.payloadEntry.Size = uint64(^uint32(0)) + 1 // exceed 32-bit diff --git a/sdk/internal/zipstream/zip_primitives.go b/sdk/internal/zipstream/zip_primitives.go index 148cf35618..b695889625 100644 --- a/sdk/internal/zipstream/zip_primitives.go +++ b/sdk/internal/zipstream/zip_primitives.go @@ -55,7 +55,7 @@ func NewSegmentMetadata(expectedCount int) *SegmentMetadata { } // AddSegment records metadata for a segment (size + CRC) without retaining payload bytes. -func (sm *SegmentMetadata) AddSegment(index int, _ []byte, originalSize uint64, originalCRC32 uint32) error { +func (sm *SegmentMetadata) AddSegment(index int, originalSize uint64, originalCRC32 uint32) error { if index < 0 { return ErrInvalidSegment } From 8c7b2e064f96b78b7d7474cf3d28ee5dd852295b Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 13 Nov 2025 11:41:21 -0800 Subject: [PATCH 13/17] lint fixes --- sdk/internal/zipstream/segment_writer_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sdk/internal/zipstream/segment_writer_test.go b/sdk/internal/zipstream/segment_writer_test.go index b00ff273e6..8636778fc5 100644 --- a/sdk/internal/zipstream/segment_writer_test.go +++ b/sdk/internal/zipstream/segment_writer_test.go @@ -115,7 +115,9 @@ func TestSegmentWriter_OutOfOrder(t *testing.T) { assert.Empty(t, bytes, "Segment %d should have no zip bytes", index) } - allBytes := append(bytes, data...) + var allBytes []byte + allBytes = append(allBytes, bytes...) + allBytes = append(allBytes, data...) segmentBytes[index] = allBytes } @@ -193,7 +195,9 @@ func TestSegmentWriter_SparseIndices_InOrder(t *testing.T) { } else { assert.Empty(t, bytes, "segment %d should have no zip bytes", index) } - totalBytes := append(bytes, data...) + var totalBytes []byte + totalBytes = append(totalBytes, bytes...) + totalBytes = append(totalBytes, data...) segmentBytes[index] = totalBytes } @@ -253,7 +257,9 @@ func TestSegmentWriter_SparseIndices_OutOfOrder(t *testing.T) { crc := crc32.ChecksumIEEE(data) bytes, err := writer.WriteSegment(ctx, index, uint64(len(data)), crc) require.NoError(t, err, "write segment %d failed", index) - totalBytes := append(bytes, data...) + var totalBytes []byte + totalBytes = append(totalBytes, bytes...) + totalBytes = append(totalBytes, data...) segmentBytes[index] = totalBytes } From 04fa7a2e008920ab788f1a682ff5ae3ec5958aa0 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 13 Nov 2025 11:55:04 -0800 Subject: [PATCH 14/17] reset examples --- examples/go.mod | 4 ++-- examples/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/go.mod b/examples/go.mod index 0adaedfae2..dd48e2eaef 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -5,10 +5,9 @@ go 1.24.0 toolchain go1.24.9 require ( - connectrpc.com/connect v1.18.1 github.com/opentdf/platform/lib/ocrypto v0.7.0 github.com/opentdf/platform/protocol/go v0.13.0 - github.com/opentdf/platform/sdk v0.10.1 + github.com/opentdf/platform/sdk v0.7.0 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 google.golang.org/grpc v1.73.0 @@ -17,6 +16,7 @@ require ( require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250603165357-b52ab10f4468.1 // indirect + connectrpc.com/connect v1.18.1 // indirect github.com/Masterminds/semver/v3 v3.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/examples/go.sum b/examples/go.sum index b05ea7e203..3b4cd41af2 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -120,8 +120,8 @@ github.com/opentdf/platform/lib/ocrypto v0.7.0 h1:uBZJXisuXU3V8681aP8FVMJkyWBrwW github.com/opentdf/platform/lib/ocrypto v0.7.0/go.mod h1:sYhoBL1bQYgQVSSNpxU13RsrE5JAk8BABT1hfr9L3j8= github.com/opentdf/platform/protocol/go v0.13.0 h1:vrOOHyhYDPzJgNenz/1g0M5nWtkOYKkPggMNHKzeMcs= github.com/opentdf/platform/protocol/go v0.13.0/go.mod h1:GRycoDGDxaz91sOvGZFWVEKJLluZFg2wM3NJmhucDHo= -github.com/opentdf/platform/sdk v0.10.1 h1:kBrTK48xle7mdGc+atlr4kDh94f6kVj+0OB76K8rozI= -github.com/opentdf/platform/sdk v0.10.1/go.mod h1:+yaTi/c/GWHZPPmO27sq2s7Tcb2P/USkK8LuW1krhI8= +github.com/opentdf/platform/sdk v0.7.0 h1:8hczDycXGY1ucdIXSrP17oW/Eyu3vsb4LEX4hc7tvVY= +github.com/opentdf/platform/sdk v0.7.0/go.mod h1:CTJR1NXeYe896M1/VN0h+1Ff54SdBtxv4z18BGTi8yk= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 61ece0bd056b2c9b887f338362710707cbcf9409 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 13 Nov 2025 13:59:15 -0800 Subject: [PATCH 15/17] fix sdk --- sdk/experimental/tdf/writer.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index 76bbe6c8ac..02d2f8af53 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -36,7 +36,6 @@ const ( // SegmentResult contains the result of writing a segment type SegmentResult struct { TDFData io.Reader // Reader for the full TDF segment (nonce + encrypted data + zip structures) - Data []byte `json:"data"` // Encrypted segment bytes Index int `json:"index"` // Segment index Hash string `json:"hash"` // Base64-encoded integrity hash PlaintextSize int64 `json:"plaintextSize"` // Original data size @@ -253,7 +252,9 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg if index > w.maxSegmentIndex { w.maxSegmentIndex = index } - seg := &Segment{} + seg := &Segment{ + Size: -1, // indicates not filled yet + } w.segments[index] = seg w.mutex.Unlock() @@ -269,9 +270,11 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg } segmentHash := string(ocrypto.Base64Encode([]byte(segmentSig))) + w.mutex.Lock() seg.Size = int64(len(data)) seg.EncryptedSize = int64(len(segmentCipher)) + int64(len(nonce)) seg.Hash = segmentHash + w.mutex.Unlock() crc := crc32.NewIEEE() _, err = crc.Write(nonce) @@ -295,7 +298,6 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg return &SegmentResult{ TDFData: reader, - Data: segmentCipher, Index: index, Hash: seg.Hash, PlaintextSize: seg.Size, @@ -540,7 +542,8 @@ func (w *Writer) getManifest(ctx context.Context, cfg *WriterFinalizeConfig) (*M var totalPlaintextSize, totalEncryptedSize int64 for _, i := range order { segment, exists := w.segments[i] - if !exists { + // if size is negative, segment was not written, finalized has been called too early + if !exists || w.segments[i].Size < 0 { return nil, 0, 0, fmt.Errorf("segment %d not written; cannot finalize", i) } if segment.Hash != "" { From 38e13f7d5b52139fa47f386b00bc7a0952efa63f Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 13 Nov 2025 14:03:27 -0800 Subject: [PATCH 16/17] benchmark --- examples/cmd/benchmark_experimental.go | 109 +++++++++++++++++++++++++ examples/go.mod | 4 +- examples/go.sum | 4 +- 3 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 examples/cmd/benchmark_experimental.go diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go new file mode 100644 index 0000000000..9e4b87a972 --- /dev/null +++ b/examples/cmd/benchmark_experimental.go @@ -0,0 +1,109 @@ +//nolint:forbidigo // We use Println here extensively because we are printing markdown. +package cmd + +import ( + "context" + "crypto/rand" + "fmt" + "sync" + "time" + + "connectrpc.com/connect" + "github.com/opentdf/platform/lib/ocrypto" + kasp "github.com/opentdf/platform/protocol/go/kas" + "github.com/opentdf/platform/protocol/go/kas/kasconnect" + "github.com/opentdf/platform/protocol/go/policy" + + "github.com/opentdf/platform/sdk/experimental/tdf" + "github.com/opentdf/platform/sdk/httputil" + "github.com/spf13/cobra" +) + +var ( + payloadSize int + segmentChunk int + testAttr = "https://example.com/attr/attr1/value/value1" +) + +func init() { + benchmarkCmd := &cobra.Command{ + Use: "benchmark-experimental-writer", + Short: "Benchmark experimental TDF writer speed", + Long: `Benchmark the experimental TDF writer with configurable payload size.`, + RunE: runExperimentalWriterBenchmark, + } + //nolint: mnd // no magic number, this is just default value for payload size + benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB + //nolint: mnd // same as above + benchmarkCmd.Flags().IntVar(&segmentChunk, "segment-chunks", 16*1024, "segment chunks ize") // Default 16 segments + ExamplesCmd.AddCommand(benchmarkCmd) +} + +func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { + payload := make([]byte, payloadSize) + _, err := rand.Read(payload) + if err != nil { + return fmt.Errorf("failed to generate random payload: %w", err) + } + + http := httputil.SafeHTTPClient() + fmt.Println("endpoint:", platformEndpoint) + serviceClient := kasconnect.NewAccessServiceClient(http, platformEndpoint) + resp, err := serviceClient.PublicKey(context.Background(), connect.NewRequest(&kasp.PublicKeyRequest{Algorithm: string(ocrypto.RSA2048Key)})) + if err != nil { + return fmt.Errorf("failed to get public key from KAS: %w", err) + } + var attrs []*policy.Value + + simpleyKey := &policy.SimpleKasKey{ + KasUri: platformEndpoint, + KasId: "id", + PublicKey: &policy.SimpleKasPublicKey{ + Kid: resp.Msg.GetKid(), + Pem: resp.Msg.GetPublicKey(), + Algorithm: policy.Algorithm_ALGORITHM_RSA_2048, + }, + } + + attrs = append(attrs, &policy.Value{Fqn: testAttr, KasKeys: []*policy.SimpleKasKey{simpleyKey}, Attribute: &policy.Attribute{Namespace: &policy.Namespace{Name: "example.com"}, Fqn: testAttr}}) + writer, err := tdf.NewWriter(context.Background(), tdf.WithDefaultKASForWriter(simpleyKey), tdf.WithInitialAttributes(attrs), tdf.WithSegmentIntegrityAlgorithm(tdf.HS256)) + if err != nil { + return fmt.Errorf("failed to create writer: %w", err) + } + i := 0 + wg := sync.WaitGroup{} + segs := len(payload) / segmentChunk + wg.Add(segs) + start := time.Now() + for i < segs { + segment := i + func() { + start := i * segmentChunk + end := min(start+segmentChunk, len(payload)) + _, err = writer.WriteSegment(context.Background(), segment, payload[start:end]) + if err != nil { + fmt.Println(err) + panic(err) + } + wg.Done() + }() + i++ + } + wg.Wait() + + end := time.Now() + result, err := writer.Finalize(context.Background()) + if err != nil { + return fmt.Errorf("failed to finalize writer: %w", err) + } + totalTime := end.Sub(start) + + fmt.Printf("# Benchmark Experimental TDF Writer Results:\n") + fmt.Printf("| Metric | Value |\n") + fmt.Printf("|--------------------|--------------|\n") + fmt.Printf("| Payload Size (B) | %d |\n", payloadSize) + fmt.Printf("| Output Size (B) | %d |\n", len(result.Data)) + fmt.Printf("| Total Time | %s |\n", totalTime) + + return nil +} diff --git a/examples/go.mod b/examples/go.mod index dd48e2eaef..0adaedfae2 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -5,9 +5,10 @@ go 1.24.0 toolchain go1.24.9 require ( + connectrpc.com/connect v1.18.1 github.com/opentdf/platform/lib/ocrypto v0.7.0 github.com/opentdf/platform/protocol/go v0.13.0 - github.com/opentdf/platform/sdk v0.7.0 + github.com/opentdf/platform/sdk v0.10.1 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 google.golang.org/grpc v1.73.0 @@ -16,7 +17,6 @@ require ( require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250603165357-b52ab10f4468.1 // indirect - connectrpc.com/connect v1.18.1 // indirect github.com/Masterminds/semver/v3 v3.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/examples/go.sum b/examples/go.sum index 3b4cd41af2..b05ea7e203 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -120,8 +120,8 @@ github.com/opentdf/platform/lib/ocrypto v0.7.0 h1:uBZJXisuXU3V8681aP8FVMJkyWBrwW github.com/opentdf/platform/lib/ocrypto v0.7.0/go.mod h1:sYhoBL1bQYgQVSSNpxU13RsrE5JAk8BABT1hfr9L3j8= github.com/opentdf/platform/protocol/go v0.13.0 h1:vrOOHyhYDPzJgNenz/1g0M5nWtkOYKkPggMNHKzeMcs= github.com/opentdf/platform/protocol/go v0.13.0/go.mod h1:GRycoDGDxaz91sOvGZFWVEKJLluZFg2wM3NJmhucDHo= -github.com/opentdf/platform/sdk v0.7.0 h1:8hczDycXGY1ucdIXSrP17oW/Eyu3vsb4LEX4hc7tvVY= -github.com/opentdf/platform/sdk v0.7.0/go.mod h1:CTJR1NXeYe896M1/VN0h+1Ff54SdBtxv4z18BGTi8yk= +github.com/opentdf/platform/sdk v0.10.1 h1:kBrTK48xle7mdGc+atlr4kDh94f6kVj+0OB76K8rozI= +github.com/opentdf/platform/sdk v0.10.1/go.mod h1:+yaTi/c/GWHZPPmO27sq2s7Tcb2P/USkK8LuW1krhI8= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From c67bcf8896e5ba4f11c600961e2c78a719553c57 Mon Sep 17 00:00:00 2001 From: Dominic Reed Date: Thu, 13 Nov 2025 14:15:46 -0800 Subject: [PATCH 17/17] benchmark experiemtnal --- examples/cmd/benchmark_experimental.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go index 9e4b87a972..f50c5cc348 100644 --- a/examples/cmd/benchmark_experimental.go +++ b/examples/cmd/benchmark_experimental.go @@ -77,7 +77,7 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { start := time.Now() for i < segs { segment := i - func() { + go func() { start := i * segmentChunk end := min(start+segmentChunk, len(payload)) _, err = writer.WriteSegment(context.Background(), segment, payload[start:end])