Add Pagefind incremental indexing

docs/userGuide/makingTheSiteSearchable.md

@@ -50,10 +50,6 @@ MarkBind now supports [Pagefind](https://pagefind.app/), a static low-bandwidth
 This is a <strong>beta</strong> feature and will be refined in future updates. To use it, you must have <code>enableSearch: true</code> in your <code>site.json</code> (this is the default).
 </box>
 
-<box type="warning">
-The Pagefind index is currently only generated during a full site build (e.g., <code>markbind build</code>). It will <strong>not</strong> repeatedly update during live reload (<code>markbind serve</code>) when you modify pages. You must restart the server (re-run <code>markbind serve</code>) or rebuild to refresh the search index.
-</box>
-
 To add the Pagefind search bar to your page, simply insert the following element where you want it to appear:
 
 ```md

packages/cli/src/util/serveUtil.ts

@@ -35,7 +35,8 @@ const addHandler = (site: any, onePagePath?: boolean) => (filePath: string): voi
   }
   Promise.resolve().then(async () => {
     if (site.isFilepathAPage(filePath) || site.isDependencyOfPage(filePath)) {
-      return site.rebuildSourceFiles();
+      await site.rebuildSourceFiles();
+      return await site.updatePagefindIndex(filePath);
     }
     return site.buildAsset(filePath);
   }).catch((err: Error) => {
@@ -59,7 +60,8 @@ const changeHandler = (site: any, onePagePath?: boolean) => (filePath: string):
       return site.reloadSiteConfig();
     }
     if (site.isDependencyOfPage(filePath)) {
-      return site.rebuildAffectedSourceFiles(filePath);
+      await site.rebuildAffectedSourceFiles(filePath);
+      return await site.updatePagefindIndex(filePath);
     }
     return site.buildAsset(filePath);
   }).catch((err: Error) => {
@@ -80,7 +82,8 @@ const removeHandler = (site: any, onePagePath?: boolean) => (filePath: string):
   }
   Promise.resolve().then(async () => {
     if (site.isFilepathAPage(filePath) || site.isDependencyOfPage(filePath)) {
-      return site.rebuildSourceFiles();
+      await site.rebuildSourceFiles();
+      return await site.indexSiteWithPagefind();
     }
     return site.removeAsset(filePath);
   }).catch((err: Error) => {

packages/core/src/Site/SiteGenerationManager.ts

@@ -84,6 +84,9 @@ export class SiteGenerationManager {
   currentOpenedPages: string[];
   toRebuild: Set<string>;
 
+  // Pagefind index state (kept in memory for serve mode for incremental updates)
+  pagefindIndex: any;
+
   constructor(rootPath: string, outputPath: string, onePagePath: string, forceReload = false,
               siteConfigPath = SITE_CONFIG_NAME, isDevMode: any, backgroundBuildMode: boolean,
               postBackgroundBuildFunc: () => void) {
@@ -105,6 +108,9 @@ export class SiteGenerationManager {
       : '';
     this.currentOpenedPages = [];
     this.toRebuild = new Set();
+
+    // Pagefind index state (kept in memory for serve mode for incremental updates)
+    this.pagefindIndex = null;
   }
 
   configure(siteAssets: SiteAssetsManager, sitePages: SitePagesManager) {
@@ -317,7 +323,15 @@ export class SiteGenerationManager {
       await this.siteAssets.copyMaterialIconsAsset();
       await this.writeSiteData();
       if (this.siteConfig.enableSearch) {
-        const indexingSucceeded = await this.indexSiteWithPagefind();
+        let indexingSucceeded: boolean;
+        if (this.onePagePath) {
+          const builtPages = this.sitePages.pages.filter(page =>
+            fs.existsSync(page.pageConfig.resultPath),
+          );
+          indexingSucceeded = await this.updatePagefindIndex(builtPages);
+        } else {
+          indexingSucceeded = await this.indexSiteWithPagefind();
+        }
         this.sitePages.pagefindIndexingSucceeded = indexingSucceeded;
       }
       this.calculateBuildTimeForGenerate(startTime, lazyWebsiteGenerationString);
@@ -440,6 +454,11 @@ export class SiteGenerationManager {
       this._setTimestampVariable();
       await this.runPageGenerationTasks([pageGenerationTask]);
       await this.writeSiteData();
+
+      if (this.siteConfig.enableSearch && this.pagefindIndex) {
+        await this.updatePagefindIndex(pagesToRebuild);
+      }
+
       SiteGenerationManager.calculateBuildTimeForRebuildPagesBeingViewed(startTime);
     } catch (err) {
       await SiteGenerationManager.rejectHandler(err, [this.tempPath, this.outputPath]);
@@ -466,6 +485,11 @@ export class SiteGenerationManager {
     const isCompleted = await this.generatePagesMarkedToRebuild();
     if (isCompleted) {
       logger.info('Background building completed!');
+
+      if (this.siteConfig.enableSearch) {
+        await this.indexSiteWithPagefind();
+      }
+
       this.postBackgroundBuildFunc();
     }
   }
@@ -872,29 +896,49 @@ export class SiteGenerationManager {
   );
 
   /**
- * Indexes all the pages of the site using pagefind.
- * @returns true if indexing succeeded and pagefind assets were written, false otherwise.
- */
+   * Initializes a new Pagefind index with proper configuration.
+   * @returns The created index object
+   */
+  private async initializePagefindIndex(): Promise<any> {
+    const { createIndex } = pagefind;
+    const pagefindConfig = this.siteConfig.pagefind || {};
+
+    const createIndexOptions: Record<string, unknown> = {
+      keepIndexUrl: true,
+      verbose: true,
+      logfile: 'debug.log',
+    };
+
+    if (pagefindConfig.exclude_selectors) {
+      createIndexOptions.excludeSelectors = pagefindConfig.exclude_selectors;
+    }
+
+    const { index } = await createIndex(createIndexOptions);
+    return index;
+  }
+
+  /**
+  * Indexes all the pages of the site using pagefind.
+  * Performs a full rebuild of the search index.
+  * @returns true if indexing succeeded and pagefind assets were written, false otherwise.
+  */
   async indexSiteWithPagefind(): Promise<boolean> {
     const startTime = new Date();
     logger.info('Creating Pagefind Search Index...');
     try {
-      const { createIndex, close } = pagefind;
-
-      const pagefindConfig = this.siteConfig.pagefind || {};
-
-      const createIndexOptions: Record<string, unknown> = {
-        keepIndexUrl: true,
-        verbose: true,
-        logfile: 'debug.log',
-      };
-
-      if (pagefindConfig.exclude_selectors) {
-        createIndexOptions.excludeSelectors = pagefindConfig.exclude_selectors;
+      // Clean up existing in-memory index if it exists
+      if (this.pagefindIndex) {
+        await this.pagefindIndex.deleteIndex();
+        this.pagefindIndex = null;
       }
 
-      const { index } = await createIndex(createIndexOptions);
+      const index = await this.initializePagefindIndex();
+      const { close } = pagefind;
+
       if (index) {
+        // Store index in memory for incremental updates in serve mode
+        this.pagefindIndex = index;
+
         // Filter pages that should be indexed (searchable !== false)
         const searchablePages = this.sitePages.pages.filter(
           page => page.pageConfig.searchable,
@@ -939,10 +983,22 @@ export class SiteGenerationManager {
         logger.info(`Pagefind indexed ${totalPageCount} pages in ${totalTime}s`);
 
         const pagefindOutputPath = path.join(this.outputPath, TEMPLATE_SITE_ASSET_FOLDER_NAME, 'pagefind');
+        // Clear output directory before writing
+        await fs.emptyDir(pagefindOutputPath);
         await fs.ensureDir(pagefindOutputPath);
         await index.writeFiles({ outputPath: pagefindOutputPath });
         logger.info(`Pagefind assets written to ${pagefindOutputPath}`);
-        await close();
+
+        // Only close the index in build/deploy mode; keep it in memory for serve mode
+        // Detect serve mode by checking if postBackgroundBuildFunc has a name (named function = serve)
+        const isServeMode = this.postBackgroundBuildFunc.name !== '';
+        const shouldClose = !isServeMode;
+
+        if (shouldClose) {
+          await close();
+          this.pagefindIndex = null;
+        }
+
         return true;
       }
       logger.error('Pagefind failed to create index');
@@ -954,6 +1010,57 @@ export class SiteGenerationManager {
     }
   }
 
+  /**
+   * Updates the search index for changed pages only (incremental update).
+   * Requires the index to be kept in memory from a prior indexSiteWithPagefind() call.
+   * @param pages Array of pages that were modified/added
+   * @returns true if update succeeded, false otherwise
+   */
+  async updatePagefindIndex(pages: Page[]): Promise<boolean> {
+    if (!this.pagefindIndex) {
+      logger.info('Pagefind index not in memory, auto-creating...');
+      this.pagefindIndex = await this.initializePagefindIndex();
+    }
+
+    const pagefindOutputPath = path.join(this.outputPath, TEMPLATE_SITE_ASSET_FOLDER_NAME, 'pagefind');
+
+    try {
+      const searchablePages = pages.filter(page => page.pageConfig.searchable);
+
+      await Promise.all(searchablePages.map(async (page) => {
+        try {
+          const content = await fs.readFile(page.pageConfig.resultPath, 'utf8');
+          const relativePath = path.relative(this.outputPath, page.pageConfig.resultPath);
+
+          return this.pagefindIndex.addHTMLFile({
+            sourcePath: relativePath,
+            content,
+          });
+        } catch (err) {
+          const pageResultPath = page.pageConfig.resultPath;
+          logger.warn(`Skipping index update for ${pageResultPath}: file not built yet`);
+          return null;
+        }
+      }));
+
+      const { files } = await this.pagefindIndex.getFiles();
+      await fs.emptyDir(pagefindOutputPath);
+
+      const pagefindFiles: { path: string; content: Uint8Array }[] = files;
+      await Promise.all(pagefindFiles.map(async (file) => {
+        const filePath = path.join(pagefindOutputPath, file.path);
+        await fs.ensureDir(path.dirname(filePath));
+        return fs.writeFile(filePath, Buffer.from(file.content));
+      }));
+
+      logger.info(`Updated Pagefind index for ${searchablePages.length} page(s)`);
+      return true;
+    } catch (error) {
+      logger.error(`Failed to update Pagefind index: ${error}`);
+      return false;
+    }
+  }
+
   async reloadSiteConfig() {
     if (this.backgroundBuildMode) {
       this.stopOngoingBuilds();

packages/core/src/Site/index.ts

@@ -86,6 +86,18 @@ export class Site {
     return this.generationManager.rebuildSourceFiles();
   }
 
+  async indexSiteWithPagefind(): Promise<boolean> {
+    return this.generationManager.indexSiteWithPagefind();
+  }
+
+  async updatePagefindIndex(filePaths: string | string[]): Promise<boolean> {
+    const paths = Array.isArray(filePaths) ? filePaths : [filePaths];
+    const pages = this.generationManager.sitePages.pages.filter(page =>
+      paths.some(p => page.pageConfig.sourcePath === p),
+    );
+    return this.generationManager.updatePagefindIndex(pages);
+  }
+
   buildAsset(filePaths: string | string[]) {
     return this.assetsManager.buildAsset(filePaths);
   }

packages/core/test/unit/Site/Site.test.ts

@@ -37,6 +37,9 @@ jest.mock('../../../src/Site/SiteGenerationManager', () => ({
     buildSourceFiles: jest.fn(),
     rebuildSourceFiles: jest.fn(),
     reloadSiteConfig: jest.fn(),
+    updatePagefindIndex: jest.fn().mockResolvedValue(true),
+    indexSiteWithPagefind: jest.fn().mockResolvedValue(true),
+    sitePages: { pages: [] },
   })),
 }));
 
@@ -120,6 +123,23 @@ test('Site rebuildSourceFiles delegates to SiteGenerationManager', () => {
   expect(site.generationManager.rebuildSourceFiles).toHaveBeenCalled();
 });
 
+test('Site updatePagefindIndex delegates to SiteGenerationManager', async () => {
+  const site = new Site(...siteArguments);
+  const mockPage = { pageConfig: { sourcePath: 'test.md', resultPath: '_site/test.html', searchable: true } };
+  const mockPages = [mockPage];
+  site.generationManager.sitePages = { pages: mockPages } as any;
+
+  await site.updatePagefindIndex('test.md');
+  expect(site.generationManager.updatePagefindIndex).toHaveBeenCalledWith(mockPages);
+});
+
+test('Site indexSiteWithPagefind delegates to SiteGenerationManager', async () => {
+  const site = new Site(...siteArguments);
+
+  await site.indexSiteWithPagefind();
+  expect(site.generationManager.indexSiteWithPagefind).toHaveBeenCalled();
+});
+
 test('Site reloadSiteConfig delegates to SiteGenerationManager', async () => {
   const site = new Site(...siteArguments);
   await site.reloadSiteConfig();