diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..2644567 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": ["Bash(tree:*)", "Bash(pnpm run:*)"], + "deny": [], + "ask": [] + } +} diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..480db49 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,103 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with +code in this repository. + +## Commands + +### Development Commands + +- `pnpm lint` - Run ESLint on the source code +- `pnpm check-types` - Run TypeScript type checking without emitting files +- `pnpm check-format` - Check code formatting with Prettier +- `pnpm format` - Auto-format code with Prettier +- `pnpm build` - Build the library (runs tsup-node, tsc for declarations, and + tsc-alias) +- `pnpm clean` - Remove dist directory and build artifacts +- `pnpm prepare` - Full build cycle (clean, check-types, and build) + +### Important Notes + +- No test runner is currently configured (test script echoes "No test + specified") +- The project uses pnpm as the package manager +- Node.js version requirement: >=20 + +## Architecture Overview + +This is a TypeScript library that provides typed abstractions for Firestore in +server environments (firebase-admin and firebase-functions). The library is +organized into three main areas: + +### Core Modules + +1. **Documents (`src/documents/`)** - Functions for single document operations + + - CRUD operations: get, set, update, delete + - Transaction variants (suffixed with `Tx`) + - Support for both typed collections and specific document references + - Returns `FsMutableDocument` which combines data with typed update/delete + methods + +2. **Collections (`src/collections/`)** - Functions for collection queries and + batch processing + + - Query functions: `getDocuments`, `getFirstDocument` + - Processing functions: `processDocuments`, `processDocumentsByChunk` + - Internal chunking for handling large collections (paginated fetching) + - Support for typed select statements that narrow both data and types + +3. **Functions (`src/functions/`)** - Cloud Functions utilities + - Helpers to extract typed data from 2nd gen cloud function events + - Functions like `getDataOnWritten`, `getBeforeAndAfterOnUpdated` + - Exported separately as `@typed-firestore/server/functions` to make + firebase-admin optional + +### Key Design Patterns + +1. **Transaction Support**: All document operations have transaction variants + (suffix `Tx`) that work within Firebase transactions. Transaction functions + return the Transaction object for chaining. + +2. **Type Narrowing with Select**: The library supports TypeScript type + narrowing when using select statements. Select must be defined separately + from the query to enable proper type inference. + +3. **Chunked Processing**: Collection operations internally use pagination to + handle unlimited documents with constant memory usage. Setting a limit on a + query disables pagination. + +4. **Mutable Documents**: The library returns `FsMutableDocument` objects + that combine: + + - `id`: Document ID + - `data`: Typed document data + - `ref`: Original Firestore reference + - `update()`/`updateWithPartial()`: Typed update methods + - `delete()`: Delete method + +5. **Path Aliases**: The codebase uses `~` as a path alias for the src directory + (configured via tsc-alias). + +### Type System + +The library provides strong typing throughout: + +- Collection references are typed as `CollectionReference` +- Documents are returned as `FsDocument` or `FsMutableDocument` +- Select statements narrow types to `Pick` +- Transaction variants have different return types for proper chaining + +### Build System + +- Uses tsup for bundling JavaScript +- TypeScript compiler for declaration files +- tsc-alias for resolving path aliases in output +- Outputs both ESM modules and TypeScript declarations +- Separate exports for main library and functions submodule + +### Code Conventions + +- Comments are written in JSDoc style, unless they are inline comments with code + on the same line. +- Use `/** ... */` for both multi-line and single-line comments. diff --git a/package.json b/package.json index f6ba4ae..24db2d4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@typed-firestore/server", - "version": "2.0.0", + "version": "2.1.0-0", "description": "Elegant, typed abstractions for Firestore in server environments", "repository": { "type": "git", diff --git a/src/collections/get-documents.ts b/src/collections/get-documents.ts index 1627d86..643a504 100644 --- a/src/collections/get-documents.ts +++ b/src/collections/get-documents.ts @@ -9,7 +9,11 @@ import { makeMutableDocument, makeMutableDocumentTx } from "~/documents"; import type { FsMutableDocument, FsMutableDocumentTx } from "~/types"; import { invariant } from "~/utils"; import { MAX_QUERY_LIMIT } from "./constants"; -import { buildQuery, getDocumentsChunked } from "./helpers"; +import { + buildQuery, + getDocumentsChunked, + getDocumentsChunkedWithLimit, +} from "./helpers"; import type { QueryBuilder, SelectedDocument } from "./types"; export type GetDocumentsOptions< @@ -35,6 +39,7 @@ export async function getDocuments< ); if (disableChunking) { + /** For limits <= MAX_QUERY_LIMIT, use a single query (existing behavior) */ invariant( limit && limit <= MAX_QUERY_LIMIT, `Limit ${String(limit)} is greater than the maximum query limit of ${String(MAX_QUERY_LIMIT)}` @@ -47,7 +52,15 @@ export async function getDocuments< doc as QueryDocumentSnapshot> ) ); + } else if (limit && limit > MAX_QUERY_LIMIT) { + /** For limits > MAX_QUERY_LIMIT, use chunking with the specified limit */ + return getDocumentsChunkedWithLimit, T>( + query, + limit, + options.chunkSize + ); } else { + /** No limit specified, get all documents using chunking */ return getDocumentsChunked, T>( query, options.chunkSize diff --git a/src/collections/helpers/build-query.ts b/src/collections/helpers/build-query.ts index e56f9af..20f6b61 100644 --- a/src/collections/helpers/build-query.ts +++ b/src/collections/helpers/build-query.ts @@ -17,17 +17,16 @@ export function buildQuery( const queryInfo = queryFn ? getQueryInfo(queryFn(ref)) : {}; const { limit, select: querySelect } = queryInfo; - invariant( - !limit || limit <= MAX_QUERY_LIMIT, - `Limit ${String(limit)} is greater than the maximum query limit of ${String(MAX_QUERY_LIMIT)}` - ); - invariant( !querySelect, "Select is not allowed to be set on the query. Use the options instead." ); - const disableChunking = isDefined(limit); + /** + * Disable chunking only for limits <= MAX_QUERY_LIMIT. For limits > + * MAX_QUERY_LIMIT, we'll use chunking + */ + const disableChunking = isDefined(limit) && limit <= MAX_QUERY_LIMIT; const baseQuery = queryFn ? queryFn(ref) : ref; diff --git a/src/collections/helpers/get-documents-chunked-with-limit.ts b/src/collections/helpers/get-documents-chunked-with-limit.ts new file mode 100644 index 0000000..d676b00 --- /dev/null +++ b/src/collections/helpers/get-documents-chunked-with-limit.ts @@ -0,0 +1,53 @@ +import type { + DocumentData, + Query, + QueryDocumentSnapshot, +} from "firebase-admin/firestore"; +import type { FsMutableDocument } from "~/types"; +import { verboseCount } from "~/utils"; +import { DEFAULT_CHUNK_SIZE, MAX_QUERY_LIMIT } from "../constants"; +import { getChunkOfDocuments } from "./get-chunk-of-documents"; + +/** + * Gets documents from a query with a specified limit, using pagination when the + * limit exceeds Firestore's maximum query limit of 1000. + */ +export async function getDocumentsChunkedWithLimit< + T extends DocumentData, + TFull extends DocumentData, +>( + query: Query, + totalLimit: number, + chunkSize = DEFAULT_CHUNK_SIZE +): Promise[]> { + const documents: FsMutableDocument[] = []; + let startAfterSnapshot: QueryDocumentSnapshot | undefined; + let remainingLimit = totalLimit; + + do { + verboseCount("Fetching chunk"); + + const currentChunkSize = Math.min( + remainingLimit, + Math.min(MAX_QUERY_LIMIT, chunkSize) + ); + + const [chunk, lastSnapshot] = await getChunkOfDocuments( + query, + startAfterSnapshot, + currentChunkSize + ); + + documents.push(...chunk); + remainingLimit -= chunk.length; + startAfterSnapshot = lastSnapshot; + + /** Stop if we've reached the limit or there are no more documents */ + if (remainingLimit <= 0 || !lastSnapshot) { + break; + } + } while (startAfterSnapshot); + + /** Ensure we don't return more than the requested limit */ + return documents.slice(0, totalLimit); +} diff --git a/src/collections/helpers/index.ts b/src/collections/helpers/index.ts index a58ee02..fc85d10 100644 --- a/src/collections/helpers/index.ts +++ b/src/collections/helpers/index.ts @@ -1,4 +1,5 @@ export * from "./build-query"; export * from "./get-all-documents-chunked"; export * from "./get-chunk-of-documents"; +export * from "./get-documents-chunked-with-limit"; export * from "./get-query-info"; diff --git a/src/collections/process-documents.ts b/src/collections/process-documents.ts index 8662f59..92c9c2a 100644 --- a/src/collections/process-documents.ts +++ b/src/collections/process-documents.ts @@ -63,6 +63,7 @@ export async function processDocuments< let errorCount = 0; if (disableChunking) { + /** For limits <= MAX_QUERY_LIMIT, use a single query (existing behavior) */ invariant( limit && limit <= MAX_QUERY_LIMIT, `Limit ${String(limit)} is greater than the maximum query limit of ${String(MAX_QUERY_LIMIT)}` @@ -85,18 +86,26 @@ export async function processDocuments< { throttleSeconds, chunkSize } ); } else { + /** For limits > MAX_QUERY_LIMIT or no limit, use chunking */ let lastDocumentSnapshot: | QueryDocumentSnapshot> | undefined; let count = 0; + const hasLimit = limit && limit > MAX_QUERY_LIMIT; + const remainingLimit = hasLimit ? limit : undefined; do { verboseCount("Processing chunk"); + /** Calculate effective chunk size based on remaining limit */ + const effectiveChunkSize = remainingLimit + ? Math.min(remainingLimit - count, Math.min(MAX_QUERY_LIMIT, chunkSize)) + : Math.min(MAX_QUERY_LIMIT, chunkSize); + const [documents, _lastDocumentSnapshot] = await getChunkOfDocuments< SelectedDocument, T - >(query, lastDocumentSnapshot, chunkSize); + >(query, lastDocumentSnapshot, effectiveChunkSize); await processInChunks( documents, @@ -115,6 +124,11 @@ export async function processDocuments< count += documents.length; lastDocumentSnapshot = _lastDocumentSnapshot; + + /** Stop if we've reached the limit */ + if (remainingLimit && count >= remainingLimit) { + break; + } } while (isDefined(lastDocumentSnapshot)); verboseLog(`Processed ${String(count)} documents`); @@ -145,13 +159,18 @@ export async function processDocumentsByChunk< ) => Promise, options: ProcessDocumentsOptions = {} ) { - const { query, disableChunking } = buildQuery(ref, queryFn, options.select); + const { query, disableChunking, limit } = buildQuery( + ref, + queryFn, + options.select + ); const { throttleSeconds = 0, chunkSize = DEFAULT_CHUNK_SIZE } = options; const errors: string[] = []; if (disableChunking) { + /** For limits <= MAX_QUERY_LIMIT, use a single query (existing behavior) */ const documents = await getDocuments(ref, queryFn, options); try { @@ -166,18 +185,26 @@ export async function processDocumentsByChunk< errors.push(getErrorMessage(err)); } } else { + /** For limits > MAX_QUERY_LIMIT or no limit, use chunking */ let lastDocumentSnapshot: | QueryDocumentSnapshot> | undefined; let count = 0; + const hasLimit = limit && limit > MAX_QUERY_LIMIT; + const remainingLimit = hasLimit ? limit : undefined; do { verboseCount("Processing chunk"); + /** Calculate effective chunk size based on remaining limit */ + const effectiveChunkSize = remainingLimit + ? Math.min(remainingLimit - count, Math.min(MAX_QUERY_LIMIT, chunkSize)) + : Math.min(MAX_QUERY_LIMIT, chunkSize); + const [documents, _lastDocumentSnapshot] = await getChunkOfDocuments< SelectedDocument, T - >(query, lastDocumentSnapshot, chunkSize); + >(query, lastDocumentSnapshot, effectiveChunkSize); try { await processInChunksByChunk( @@ -193,6 +220,11 @@ export async function processDocumentsByChunk< count += documents.length; lastDocumentSnapshot = _lastDocumentSnapshot; + + /** Stop if we've reached the limit */ + if (remainingLimit && count >= remainingLimit) { + break; + } } while (isDefined(lastDocumentSnapshot)); verboseLog(`Processed ${String(count)} documents`);