From 0c9a919785d199509963970818a8d1cd7fd795cf Mon Sep 17 00:00:00 2001 From: nahuel11500 <134035119+nahuel11500@users.noreply.github.com> Date: Fri, 10 Oct 2025 06:52:13 +0200 Subject: [PATCH] feat: implement chunked processing for Notion API blocks and add tests --- internal/notion/client.go | 86 ++++++++++++++++++++++++++++------ internal/notion/client_test.go | 68 +++++++++++++++++++++++++++ internal/notion/types.go | 5 +- 3 files changed, 143 insertions(+), 16 deletions(-) create mode 100644 internal/notion/client_test.go diff --git a/internal/notion/client.go b/internal/notion/client.go index 8bfb211..572df65 100644 --- a/internal/notion/client.go +++ b/internal/notion/client.go @@ -19,6 +19,9 @@ const ( MaxRetries = 3 BaseBackoff = 1 * time.Second MaxBackoff = 16 * time.Second + // BlockChunkSize defines the maximum number of blocks to send in a single API call + // Notion's limit is 100, but we use 50 for better reliability with large documents + BlockChunkSize = 50 ) // Client handles Notion API interactions @@ -61,39 +64,54 @@ func (c *Client) formatPageID(pageID string) string { cleaned[0:8], cleaned[8:12], cleaned[12:16], cleaned[16:20], cleaned[20:32]) } -// AppendBlockChildren appends blocks to a page or block -func (c *Client) AppendBlockChildren(ctx context.Context, blockID string, blocks []Block) error { - formattedID := c.formatPageID(blockID) +// processBlocksInChunks processes blocks in chunks to respect Notion's API limits +// Notion's API has a limit of 100 blocks per request. This function splits +// blocks into smaller chunks and processes them sequentially with a small +// delay between chunks to be gentle on the API. +func (c *Client) processBlocksInChunks(ctx context.Context, blocks []Block, processFn func(ctx context.Context, chunk []Block) error) error { + if len(blocks) == 0 { + return nil + } - // Split blocks into chunks of 25 for better reliability with large documents - chunkSize := 25 - for i := 0; i < len(blocks); i += chunkSize { - end := i + chunkSize + for i := 0; i < len(blocks); i += BlockChunkSize { + end := i + BlockChunkSize if end > len(blocks) { end = len(blocks) } chunk := blocks[i:end] - req := AppendBlockChildrenRequest{Children: chunk} - - if err := c.makeRequest(ctx, "PATCH", fmt.Sprintf("/blocks/%s/children", formattedID), req, nil); err != nil { - return fmt.Errorf("failed to append blocks (chunk %d-%d): %w", i+1, end, err) + if err := processFn(ctx, chunk); err != nil { + return fmt.Errorf("failed to process blocks (chunk %d-%d): %w", i+1, end, err) } if c.verbose { - fmt.Fprintf(os.Stderr, "Uploaded %d blocks (chunk %d-%d)\n", len(chunk), i+1, end) + fmt.Fprintf(os.Stderr, "Processed %d blocks (chunk %d-%d)\n", len(chunk), i+1, end) } // Small pause between chunks to be nice to the API if end < len(blocks) { - time.Sleep(100 * time.Millisecond) + time.Sleep(10 * time.Millisecond) } } return nil } +// AppendBlockChildren appends blocks to a page or block +// Blocks are automatically split into chunks to respect Notion's 100-block limit per API call. +// Uses a chunk size of 50 for better reliability with large documents. +func (c *Client) AppendBlockChildren(ctx context.Context, blockID string, blocks []Block) error { + formattedID := c.formatPageID(blockID) + + return c.processBlocksInChunks(ctx, blocks, func(ctx context.Context, chunk []Block) error { + req := AppendBlockChildrenRequest{Children: chunk} + return c.makeRequest(ctx, "PATCH", fmt.Sprintf("/blocks/%s/children", formattedID), req, nil) + }) +} + // CreatePage creates a new page under a parent page +// The page is created first without children, then blocks are appended in chunks +// to avoid Notion's 100-block limit per API call. func (c *Client) CreatePage(ctx context.Context, parentID, title string, blocks []Block) (*PageResponse, error) { formattedParentID := c.formatPageID(parentID) titleText := []RichText{{ @@ -101,6 +119,7 @@ func (c *Client) CreatePage(ctx context.Context, parentID, title string, blocks Text: &Text{Content: title}, }} + // Create the page without children first to avoid the 100-block limit req := CreatePageRequest{ Parent: Parent{ Type: "page_id", @@ -109,7 +128,7 @@ func (c *Client) CreatePage(ctx context.Context, parentID, title string, blocks Properties: PageProperties{ Title: TitleProperty{Title: titleText}, }, - Children: blocks, + // Don't include children in the initial creation } var resp PageResponse @@ -117,6 +136,45 @@ func (c *Client) CreatePage(ctx context.Context, parentID, title string, blocks return nil, fmt.Errorf("failed to create page: %w", err) } + // If there are blocks to add, append them in chunks after page creation + if len(blocks) > 0 { + if err := c.AppendBlockChildren(ctx, resp.ID, blocks); err != nil { + return nil, fmt.Errorf("failed to add content to page: %w", err) + } + } + + return &resp, nil +} + +// CreatePageInDatabase creates a new page in a database +// The page is created first without children, then blocks are appended in chunks +// to avoid Notion's 100-block limit per API call. +// Note: When creating in a database, the properties must match the database schema +func (c *Client) CreatePageInDatabase(ctx context.Context, databaseID string, properties PageProperties, blocks []Block) (*PageResponse, error) { + formattedDatabaseID := c.formatPageID(databaseID) + + // Create the page without children first to avoid the 100-block limit + req := CreatePageRequest{ + Parent: Parent{ + Type: "database_id", + DatabaseID: formattedDatabaseID, + }, + Properties: properties, + // Don't include children in the initial creation + } + + var resp PageResponse + if err := c.makeRequest(ctx, "POST", "/pages", req, &resp); err != nil { + return nil, fmt.Errorf("failed to create page in database: %w", err) + } + + // If there are blocks to add, append them in chunks after page creation + if len(blocks) > 0 { + if err := c.AppendBlockChildren(ctx, resp.ID, blocks); err != nil { + return nil, fmt.Errorf("failed to add content to page: %w", err) + } + } + return &resp, nil } diff --git a/internal/notion/client_test.go b/internal/notion/client_test.go new file mode 100644 index 0000000..5b44e4a --- /dev/null +++ b/internal/notion/client_test.go @@ -0,0 +1,68 @@ +package notion + +import ( + "context" + "testing" +) + +func TestProcessBlocksInChunks(t *testing.T) { + client := &Client{verbose: false} + + // Test with empty blocks + err := client.processBlocksInChunks(context.Background(), []Block{}, func(ctx context.Context, chunk []Block) error { + t.Error("Function should not be called with empty blocks") + return nil + }) + if err != nil { + t.Errorf("Expected nil error for empty blocks, got %v", err) + } + + // Test with blocks smaller than chunk size + smallBlocks := make([]Block, 25) + for i := range smallBlocks { + smallBlocks[i] = Block{Type: "paragraph"} + } + + callCount := 0 + err = client.processBlocksInChunks(context.Background(), smallBlocks, func(ctx context.Context, chunk []Block) error { + callCount++ + if len(chunk) != 25 { + t.Errorf("Expected chunk size 25, got %d", len(chunk)) + } + return nil + }) + if err != nil { + t.Errorf("Expected nil error, got %v", err) + } + if callCount != 1 { + t.Errorf("Expected 1 call, got %d", callCount) + } + + // Test with blocks larger than chunk size + largeBlocks := make([]Block, 126) // This simulates the original problem + for i := range largeBlocks { + largeBlocks[i] = Block{Type: "paragraph"} + } + + callCount = 0 + totalProcessed := 0 + err = client.processBlocksInChunks(context.Background(), largeBlocks, func(ctx context.Context, chunk []Block) error { + callCount++ + totalProcessed += len(chunk) + if len(chunk) > BlockChunkSize { + t.Errorf("Chunk size %d exceeds maximum %d", len(chunk), BlockChunkSize) + } + return nil + }) + if err != nil { + t.Errorf("Expected nil error, got %v", err) + } + + expectedCalls := (126 + BlockChunkSize - 1) / BlockChunkSize // Ceiling division + if callCount != expectedCalls { + t.Errorf("Expected %d calls, got %d", expectedCalls, callCount) + } + if totalProcessed != 126 { + t.Errorf("Expected to process 126 blocks, processed %d", totalProcessed) + } +} diff --git a/internal/notion/types.go b/internal/notion/types.go index 1765f2d..3c8e2d7 100644 --- a/internal/notion/types.go +++ b/internal/notion/types.go @@ -120,8 +120,9 @@ type CreatePageRequest struct { // Parent specifies the parent of a page type Parent struct { - Type string `json:"type"` - PageID string `json:"page_id,omitempty"` + Type string `json:"type"` + PageID string `json:"page_id,omitempty"` + DatabaseID string `json:"database_id,omitempty"` } // PageProperties contains page metadata