fix(github-workflow): prune emptied sync chunk dirs (#13002) (#13009)

neo-gpt · web-flow · commit 9e995dc6e216 · 2026-06-12T21:21:58.000+02:00
diff --git a/ai/services/github-workflow/shared/pruneEmptyDirs.mjs b/ai/services/github-workflow/shared/pruneEmptyDirs.mjs
@@ -0,0 +1,40 @@
+import fs   from 'fs/promises';
+import path from 'path';
+
+/**
+ * @module ai/services/github-workflow/shared/pruneEmptyDirs
+ * @summary Root-preserving cleanup for empty content chunk directories after sync moves.
+ *
+ * Syncers can empty active or archive chunk directories when an item moves to a new
+ * bucket or is dropped. Pipeline worktrees can also retain empty active chunks from
+ * earlier runs because Git cannot represent them. This helper removes empty descendants
+ * while deliberately preserving the configured root directory, so downstream sync/index
+ * code can still treat the root as an existing collection boundary.
+ *
+ * @param {String} root Absolute or repository-relative directory whose empty descendants should be pruned.
+ * @returns {Promise<void>}
+ */
+export default async function pruneEmptyDirs(root) {
+    let entries;
+
+    try {
+        entries = await fs.readdir(root, {withFileTypes: true});
+    } catch {
+        return; // root absent means there is nothing to prune.
+    }
+
+    for (const entry of entries) {
+        if (!entry.isDirectory()) continue;
+
+        const child = path.join(root, entry.name);
+        await pruneEmptyDirs(child);
+
+        try {
+            if ((await fs.readdir(child)).length === 0) {
+                await fs.rmdir(child);
+            }
+        } catch {
+            // Directory changed or disappeared during cleanup; the next sync can retry.
+        }
+    }
+}
diff --git a/ai/services/github-workflow/sync/DiscussionSyncer.mjs b/ai/services/github-workflow/sync/DiscussionSyncer.mjs
@@ -13,6 +13,7 @@ import {
     createContentIndexEntry,
     updateContentIndex
 } from '../shared/contentIndex.mjs';
+import pruneEmptyDirs               from '../shared/pruneEmptyDirs.mjs';
 import {verifyDiscussionFrontmatter} from './verifyFrontmatterIntegrity.mjs';
 
 const issueSyncConfig = aiConfig.issueSync;
@@ -289,6 +290,7 @@ class DiscussionSyncer extends Base {
         }
 
         const planBuckets = this.#planBuckets(metadata, allDiscussions);
+        let shouldPruneEmptyDirs = false;
 
         for (const discussion of allDiscussions) {
             try {
@@ -369,6 +371,7 @@ class DiscussionSyncer extends Base {
                 if (oldAbsolutePath && oldAbsolutePath !== targetPath) {
                     try {
                         await fs.unlink(oldAbsolutePath);
+                        shouldPruneEmptyDirs = true;
                         logger.debug(`📦 Moved Discussion #${discussion.number}: ${oldAbsolutePath} → ${targetPath}`);
                     } catch (e) {
                         // File might not exist
@@ -387,6 +390,11 @@ class DiscussionSyncer extends Base {
             }
         }
 
+        await pruneEmptyDirs(issueSyncConfig.discussionsDir);
+        if (shouldPruneEmptyDirs) {
+            await pruneEmptyDirs(path.join(issueSyncConfig.archiveRoot, 'discussions'));
+        }
+
         // Cache for the main orchestrator to merge
         metadata.discussions = {};
         const indexEntries = [];
diff --git a/ai/services/github-workflow/sync/IssueSyncer.mjs b/ai/services/github-workflow/sync/IssueSyncer.mjs
@@ -13,6 +13,7 @@ import {FETCH_ISSUE_TIMELINE_PAGE, FETCH_ISSUES_FOR_SYNC, FETCH_SINGLE_ISSUE} fr
 import {GET_ISSUE_ID, UPDATE_ISSUE}                                                                        from '../queries/mutations.mjs';
 import contentPath                                      from '../shared/contentPath.mjs';
 import {createContentIndexEntry, updateContentIndex}    from '../shared/contentIndex.mjs';
+import pruneEmptyDirs                                  from '../shared/pruneEmptyDirs.mjs';
 
 const issueSyncConfig = aiConfig.issueSync;
 const lineBreaksRegex = /[\r\n]+/g;
@@ -608,6 +609,7 @@ class IssueSyncer extends Base {
         };
 
         const indexMutations = {upsert: [], remove: []};
+        let shouldPruneEmptyDirs = false;
 
         const planBuckets = this.#planBuckets(metadata, allIssues);
 
@@ -650,6 +652,7 @@ class IssueSyncer extends Base {
                     try {
                         const oldPath = this.#resolvePath(oldPathRelative);
                         await fs.unlink(oldPath);
+                        shouldPruneEmptyDirs = true;
                         logger.debug(`🗑️ Removed dropped issue #${issueNumber}: ${oldPath}`);
                     } catch (e) { /* File might not exist */ }
                 }
@@ -690,10 +693,11 @@ class IssueSyncer extends Base {
                     stats.pulled.moved++;
                     try {
                         await fs.rename(oldAbsolutePath, targetPath);
+                        shouldPruneEmptyDirs = true;
                         logger.debug(`📦 Moved #${issueNumber}: ${oldAbsolutePath} → ${targetPath}`);
                     } catch (e) {
                         logger.warn(`Could not rename #${issueNumber}, falling back to write. Error: ${e.message}`);
-                        await fs.unlink(oldAbsolutePath).catch(() => {});
+                        await fs.unlink(oldAbsolutePath).then(() => { shouldPruneEmptyDirs = true; }).catch(() => {});
                     }
                 } else {
                     stats.pulled.updated++;
@@ -789,6 +793,11 @@ class IssueSyncer extends Base {
             stats.pulled.issues.push(...refetchStats.refetched.issues);
         }
 
+        await pruneEmptyDirs(issueSyncConfig.issuesDir);
+        if (shouldPruneEmptyDirs) {
+            await pruneEmptyDirs(path.join(issueSyncConfig.archiveRoot, 'issues'));
+        }
+
         try {
             await updateContentIndex(issueSyncConfig, indexMutations);
         } catch (e) {
@@ -1023,6 +1032,7 @@ class IssueSyncer extends Base {
         logger.info('🔄 Reconciling closed issue locations...');
 
         const stats = { count: 0, issues: [] };
+        let shouldPruneEmptyDirs = false;
 
         // Ensure releases are loaded
         if (!ReleaseNotesSyncer.sortedReleases || ReleaseNotesSyncer.sortedReleases.length === 0) {
@@ -1078,6 +1088,7 @@ class IssueSyncer extends Base {
 
                     // Move the file
                     await fs.rename(currentAbsolutePath, correctPath);
+                    shouldPruneEmptyDirs = true;
 
                     // Update metadata with relative path
                     metadata.issues[issueNumber].path = this.#relativePath(correctPath);
@@ -1092,6 +1103,8 @@ class IssueSyncer extends Base {
             }
         }
 
+        await pruneEmptyDirs(issueSyncConfig.issuesDir);
+
         if (stats.count > 0) {
             logger.info(`📦 Archived ${stats.count} closed issue(s)`);
         } else {
@@ -1195,37 +1208,13 @@ class IssueSyncer extends Base {
         await updateContentIndex(issueSyncConfig, {upsert: upserts});
 
         // Prune chunk/version directories emptied by the relocation.
-        await this.#pruneEmptyDirs(path.join(issueSyncConfig.archiveRoot, 'issues'));
-        await this.#pruneEmptyDirs(issueSyncConfig.issuesDir);
+        await pruneEmptyDirs(path.join(issueSyncConfig.archiveRoot, 'issues'));
+        await pruneEmptyDirs(issueSyncConfig.issuesDir);
 
         logger.info(`[REBUCKET] moved ${moves.length} issue(s), ${unchanged} unchanged. Distribution: ${JSON.stringify(byVersion)}`);
         return summary;
     }
 
-    /**
-     * Recursively removes now-empty directories under `root` (after a re-bucket relocation leaves
-     * source chunk/version folders empty). Children are pruned before parents; `root` is never removed.
-     * @param {string} root Absolute directory to prune within.
-     * @private
-     */
-    async #pruneEmptyDirs(root) {
-        let entries;
-        try {
-            entries = await fs.readdir(root, {withFileTypes: true});
-        } catch {
-            return; // root absent — nothing to prune
-        }
-
-        for (const entry of entries) {
-            if (!entry.isDirectory()) continue;
-            const child = path.join(root, entry.name);
-            await this.#pruneEmptyDirs(child);
-            try {
-                if ((await fs.readdir(child)).length === 0) await fs.rmdir(child);
-            } catch { /* race / already removed */ }
-        }
-    }
-
     /**
      * Recursively scans the configured issue directory to find all local .md issue files.
      * This operation is intentionally limited to the active issues directory as a performance
diff --git a/ai/services/github-workflow/sync/PullRequestSyncer.mjs b/ai/services/github-workflow/sync/PullRequestSyncer.mjs
@@ -12,6 +12,7 @@ import ReleaseNotesSyncer         from './ReleaseNotesSyncer.mjs';
 import {FETCH_PULL_REQUESTS_FOR_SYNC} from '../queries/pullRequestQueries.mjs';
 import contentPath                from '../shared/contentPath.mjs';
 import {createContentIndexEntry, updateContentIndex} from '../shared/contentIndex.mjs';
+import pruneEmptyDirs             from '../shared/pruneEmptyDirs.mjs';
 
 const issueSyncConfig = aiConfig.issueSync;
 const pullRequestConfig = aiConfig.pullRequest;
@@ -282,6 +283,7 @@ class PullRequestSyncer extends Base {
 
         const cachedPulls = metadata.pulls || {};
         const planBuckets = this.#planBuckets(metadata, allPullRequests);
+        let shouldPruneEmptyDirs = false;
 
         for (const pr of allPullRequests) {
             try {
@@ -354,6 +356,7 @@ class PullRequestSyncer extends Base {
                 if (oldAbsolutePath && oldAbsolutePath !== targetPath) {
                     try {
                         await fs.unlink(oldAbsolutePath);
+                        shouldPruneEmptyDirs = true;
                         logger.debug(`📦 Moved PR #${pr.number}: ${oldAbsolutePath} → ${targetPath}`);
                     } catch (e) {
                         // File might not exist
@@ -372,6 +375,11 @@ class PullRequestSyncer extends Base {
             }
         }
 
+        await pruneEmptyDirs(issueSyncConfig.pullsDir);
+        if (shouldPruneEmptyDirs) {
+            await pruneEmptyDirs(path.join(issueSyncConfig.archiveRoot, 'pulls'));
+        }
+
         // Cache for the main orchestrator to merge
         metadata.pulls = {};
         const indexEntries = [];
diff --git a/test/playwright/unit/ai/services/github-workflow/DiscussionSyncer.spec.mjs b/test/playwright/unit/ai/services/github-workflow/DiscussionSyncer.spec.mjs
@@ -275,6 +275,48 @@ test.describe('Neo.ai.services.github-workflow.sync.DiscussionSyncer', () => {
         expect(content).toMatch(/^closedAt: '2026-05-01T00:00:00Z'$/m);
     });
 
+    test('syncDiscussions prunes emptied active chunk directories after archive moves (#13002)', async () => {
+        const discussion = buildDiscussion(24006, {
+            closed  : true,
+            closedAt: '2026-05-01T00:00:00Z'
+        });
+        const oldPath = path.join(aiConfig.issueSync.discussionsDir, 'chunk-77', 'discussion-24006.md');
+        const oldRel  = path.relative(aiConfig.projectRoot, oldPath);
+
+        await fs.ensureDir(path.dirname(oldPath));
+        await fs.writeFile(oldPath, 'OLD DISCUSSION CONTENT', 'utf8');
+
+        GraphqlService.query = async () => ({
+            repository: {
+                discussions: {
+                    nodes   : [discussion],
+                    pageInfo: {hasNextPage: false, endCursor: null}
+                }
+            }
+        });
+
+        const metadata = {
+            discussions: {
+                24006: {
+                    closed     : false,
+                    closedAt   : null,
+                    contentHash: 'old-hash',
+                    path       : oldRel
+                }
+            }
+        };
+
+        const stats      = await DiscussionSyncer.syncDiscussions(metadata);
+        const targetPath = path.join(aiConfig.issueSync.archiveRoot, 'discussions', 'v13.0.0', 'chunk-1', 'discussion-24006.md');
+
+        expect(stats.synced).toEqual([24006]);
+        await expect(fs.pathExists(targetPath)).resolves.toBe(true);
+        await expect(fs.pathExists(oldPath)).resolves.toBe(false);
+        await expect(fs.pathExists(path.dirname(oldPath))).resolves.toBe(false);
+        await expect(fs.pathExists(aiConfig.issueSync.discussionsDir)).resolves.toBe(true);
+        expect(metadata.discussions[24006].path).toBe(path.relative(aiConfig.projectRoot, targetPath));
+    });
+
     test('delta cutoff stops discussion pagination once a batch predates the cached high-water mark (#12190)', async () => {
         // Mirror of the PR/issue delta: order UPDATED_AT DESC + stop at the cached high-water mark.
         const metadata = {
diff --git a/test/playwright/unit/ai/services/github-workflow/IssueSyncer.spec.mjs b/test/playwright/unit/ai/services/github-workflow/IssueSyncer.spec.mjs
@@ -401,6 +401,47 @@ test.describe('Neo.ai.services.github-workflow.sync.IssueSyncer', () => {
         }
     });
 
+    test('reconcileClosedIssueLocations prunes emptied active chunk directories after archiving (#13002)', async () => {
+        const originalSorted = ReleaseNotesSyncer.sortedReleases;
+        const issueNumber    = 6003;
+        const oldAbs         = path.join(issueSyncConfig.issuesDir, 'chunk-77', `issue-${issueNumber}.md`);
+        const oldRel         = path.relative(aiConfig.projectRoot, oldAbs);
+
+        await fs.ensureDir(path.dirname(oldAbs));
+        await fs.writeFile(oldAbs, 'CLOSED ISSUE CONTENT', 'utf8');
+
+        ReleaseNotesSyncer.sortedReleases = [{tagName: 'v13.0.0', publishedAt: '2026-05-10T00:00:00Z'}];
+
+        const metadata = {
+            issues: {
+                [issueNumber]: {
+                    state        : 'CLOSED',
+                    path         : oldRel,
+                    updatedAt    : '2026-05-02T00:00:00Z',
+                    closedAt     : '2026-05-01T00:00:00Z',
+                    milestone    : null,
+                    title        : 'Closed issue ready for archival',
+                    contentHash  : 'hash',
+                    commentsTotal: 0
+                }
+            }
+        };
+
+        try {
+            const stats     = await IssueSyncer.reconcileClosedIssueLocations(metadata);
+            const targetAbs = path.join(issueSyncConfig.archiveRoot, 'issues', 'v13.0.0', 'chunk-1', `issue-${issueNumber}.md`);
+
+            expect(stats.count).toBe(1);
+            expect(metadata.issues[issueNumber].path).toBe(path.relative(aiConfig.projectRoot, targetAbs));
+            await expect(fs.pathExists(targetAbs)).resolves.toBe(true);
+            await expect(fs.pathExists(oldAbs)).resolves.toBe(false);
+            await expect(fs.pathExists(path.dirname(oldAbs))).resolves.toBe(false);
+            await expect(fs.pathExists(issueSyncConfig.issuesDir)).resolves.toBe(true);
+        } finally {
+            ReleaseNotesSyncer.sortedReleases = originalSorted;
+        }
+    });
+
     test('pullFromGitHub enforces sealed-chunk archive semantics', async () => {
         const mockIssue = buildMockIssue({
             number       : 42044,
diff --git a/test/playwright/unit/ai/services/github-workflow/PullRequestSyncer.spec.mjs b/test/playwright/unit/ai/services/github-workflow/PullRequestSyncer.spec.mjs
@@ -206,6 +206,47 @@ test.describe('Neo.ai.services.github-workflow.sync.PullRequestSyncer', () => {
         await expect(fs.pathExists(cutArchivePath)).resolves.toBe(true);
     });
 
+    test('syncPullRequests prunes emptied active chunk directories after archive moves (#13002)', async () => {
+        const prNumber = 3291;
+        const pr       = buildPullRequest(prNumber);
+        const oldPath  = path.join(aiConfig.issueSync.pullsDir, 'chunk-77', `pr-${prNumber}.md`);
+        const oldRel   = path.relative(aiConfig.projectRoot, oldPath);
+
+        ReleaseNotesSyncer.sortedReleases = [{tagName: 'v13.0.0', publishedAt: '2026-05-10T00:00:00Z'}];
+
+        await fs.ensureDir(path.dirname(oldPath));
+        await fs.writeFile(oldPath, 'OLD PR CONTENT', 'utf8');
+
+        GraphqlService.query = async () => ({
+            repository: {
+                pullRequests: {
+                    nodes   : [pr],
+                    pageInfo: {hasNextPage: false, endCursor: null}
+                }
+            }
+        });
+
+        const metadata = {
+            pulls: {
+                [prNumber]: {
+                    state    : 'OPEN',
+                    updatedAt: '2026-05-01T00:00:00Z',
+                    path     : oldRel
+                }
+            }
+        };
+
+        const stats      = await PullRequestSyncer.syncPullRequests(metadata);
+        const targetPath = path.join(aiConfig.issueSync.archiveRoot, 'pulls', 'v13.0.0', 'chunk-1', `pr-${prNumber}.md`);
+
+        expect(stats.synced).toEqual([prNumber]);
+        await expect(fs.pathExists(targetPath)).resolves.toBe(true);
+        await expect(fs.pathExists(oldPath)).resolves.toBe(false);
+        await expect(fs.pathExists(path.dirname(oldPath))).resolves.toBe(false);
+        await expect(fs.pathExists(aiConfig.issueSync.pullsDir)).resolves.toBe(true);
+        expect(metadata.pulls[prNumber].path).toBe(path.relative(aiConfig.projectRoot, targetPath));
+    });
+
     test('delta cutoff stops PR pagination once a batch predates the cached high-water mark (#12190)', async () => {
         // The `pullRequests` connection has no server-side `since`, so the syncer orders UPDATED_AT
         // DESC and stops paginating at the cached high-water mark. Pre-fix it scanned the full corpus.
