Address llms external filtering review

Author: xavierandueza

src/commands/import.js

@@ -3336,14 +3336,14 @@ function narrowToDocsSubtreeIfNeeded(llms, sourceUrl, hits) {
  * enter the stub tree (skeleton has zero visibility of OAS files). The caller
  * downloads the returned URLs into the staged `oas/` dir.
  */
-function extractOasJsonUrlsFromParsed(parsed) {
-  const captured = []
+function dropParsedItems(parsed, shouldDrop) {
+  const dropped = []
   const keptSections = []
   for (const section of parsed.sections) {
     const keptItems = []
     for (const item of section.items) {
-      if (isOasJsonUrl(item.url)) {
-        captured.push({ url: item.url, text: item.text || null })
+      if (shouldDrop(item)) {
+        dropped.push(item)
         continue
       }
       keptItems.push(item)
@@ -3354,7 +3354,14 @@ function extractOasJsonUrlsFromParsed(parsed) {
     }
   }
   parsed.sections = keptSections
-  return captured
+  return dropped
+}
+
+function extractOasJsonUrlsFromParsed(parsed) {
+  return dropParsedItems(parsed, (item) => isOasJsonUrl(item.url)).map((item) => ({
+    url: item.url,
+    text: item.text || null,
+  }))
 }
 
 /**
@@ -3363,45 +3370,11 @@ function extractOasJsonUrlsFromParsed(parsed) {
  * items dropped.
  */
 function dropAssetItemsFromParsed(parsed) {
-  let dropped = 0
-  const keptSections = []
-  for (const section of parsed.sections) {
-    const keptItems = []
-    for (const item of section.items) {
-      if (isAssetOrMetaUrl(item.url)) {
-        dropped++
-        continue
-      }
-      keptItems.push(item)
-    }
-    if (keptItems.length > 0) {
-      section.items = keptItems
-      keptSections.push(section)
-    }
-  }
-  parsed.sections = keptSections
-  return dropped
+  return dropParsedItems(parsed, (item) => isAssetOrMetaUrl(item.url)).length
 }
 
 function dropExternalItemsFromParsed(parsed, allowedOrigins) {
-  let dropped = 0
-  const keptSections = []
-  for (const section of parsed.sections) {
-    const keptItems = []
-    for (const item of section.items) {
-      if (isExternalToAllowedOrigins(item.url, allowedOrigins)) {
-        dropped++
-        continue
-      }
-      keptItems.push(item)
-    }
-    if (keptItems.length > 0) {
-      section.items = keptItems
-      keptSections.push(section)
-    }
-  }
-  parsed.sections = keptSections
-  return dropped
+  return dropParsedItems(parsed, (item) => isExternalToAllowedOrigins(item.url, allowedOrigins)).length
 }
 
 /**
@@ -4511,7 +4484,7 @@ function makeIconPicker() {
   }
 }
 
-export const __test__ = { discoverLlmsTxt, mergeValidHits, dropExternalItemsFromParsed }
+export const __test__ = { discoverLlmsTxt, mergeValidHits, produceOrganizedForSource }
 
 function formatDuration(ms) {
   const safe = Math.max(0, ms)

src/commands/import.test.js

@@ -26,32 +26,61 @@ function mergedItems(merged) {
   return merged.parsed.sections.flatMap((section) => section.items.map((item) => ({ text: item.text, url: item.url })))
 }
 
-test('llms drops external page links before organization', () => {
-  const parsed = {
-    sections: [
-      {
-        title: 'Guides',
-        items: [
-          { text: 'Overview', url: 'https://docs.example.com/overview.md' },
-          { text: 'Canonical', url: 'https://canonical-docs.example.com/install.md' },
-          { text: 'Platform Status', url: 'https://status.example.com/' },
-          { text: 'GitHub', url: 'https://github.com/example/project' },
-        ],
-      },
-      {
-        title: 'External Only',
-        items: [{ text: 'Support', url: 'https://support.example.com/' }],
-      },
-    ],
+function organizedPages(organized) {
+  const pages = []
+  const visit = (page) => {
+    pages.push(page)
+    for (const child of page.pages || []) visit(child)
   }
+  for (const category of organized.categories || []) {
+    for (const page of category.pages || []) visit(page)
+  }
+  return pages
+}
+
+test('llms import drops external page links before organization', async () => {
+  mockLlmsFetch({
+    'https://docs.example.com/llms.txt': `# Docs
+
+## Guides
+- [Overview](/overview.md)
+- [Install](/install.md)
+- [Configure](/configure.md)
+- [Deploy](/deploy.md)
+- [Operate](/operate.md)
+
+## Reference
+- [CLI](/cli.md)
+- [SDK](/sdk.md)
+- [API](/api.md)
+- [Auth](/auth.md)
+- [Webhooks](/webhooks.md)
+
+## Admin
+- [Users](/users.md)
+- [Teams](/teams.md)
+- [Billing](/billing.md)
+- [Audit Logs](/audit-logs.md)
+- [Settings](/settings.md)
+
+## External
+- [Platform Status](https://status.example.com/)
+- [GitHub](https://github.com/example/project)
+`,
+  })
 
-  const dropped = __test__.dropExternalItemsFromParsed(parsed, new Set(['https://docs.example.com', 'https://canonical-docs.example.com']))
+  const organized = await __test__.produceOrganizedForSource(
+    new URL('https://docs.example.com/'),
+    { model: 'test' },
+    async (_label, fn) => fn(),
+  )
+  const pages = organizedPages(organized)
+  const urls = pages.map((page) => page.url).filter(Boolean)
 
-  assert.equal(dropped, 3)
-  assert.deepEqual(mergedItems({ parsed }), [
-    { text: 'Overview', url: 'https://docs.example.com/overview.md' },
-    { text: 'Canonical', url: 'https://canonical-docs.example.com/install.md' },
-  ])
+  assert(urls.includes('https://docs.example.com/overview.md'))
+  assert(urls.includes('https://docs.example.com/cli.md'))
+  assert(!urls.some((url) => url.includes('status.example.com')))
+  assert(!urls.some((url) => url.includes('github.com/example/project')))
 })
 
 test('llms discovery and merge resolve nested files, dedupe canonical pages, and preserve distinct page states', async () => {