repo sync

script/search/lunr-search-index.js

@@ -78,16 +78,22 @@ export default class LunrIndex {
     return Object.fromEntries(this.records.map((record) => [record.objectID, record]))
   }
 
-  async write(outDirectory = path.posix.join(__dirname, '../../lib/search/indexes')) {
+  async write({
+    outDirectory = path.posix.join(__dirname, '../../lib/search/indexes'),
+    compressFiles = true,
+  }) {
     this.build()
 
     // Write the parsed records
     await Promise.resolve(this.recordsObject)
       .then(JSON.stringify)
-      .then(compress)
+      .then((str) => (compressFiles ? compress(str) : str))
       .then((content) =>
         fs.writeFile(
-          path.join(outDirectory, `${this.name}-records.json.br`),
+          path.join(
+            outDirectory,
+            compressFiles ? `${this.name}-records.json.br` : `${this.name}-records.json`
+          ),
           content
           // Do not set to 'utf8'
         )
@@ -96,10 +102,10 @@ export default class LunrIndex {
     // Write the index
     await Promise.resolve(this.index)
       .then(JSON.stringify)
-      .then(compress)
+      .then((str) => (compressFiles ? compress(str) : str))
       .then((content) =>
         fs.writeFile(
-          path.join(outDirectory, `${this.name}.json.br`),
+          path.join(outDirectory, compressFiles ? `${this.name}.json.br` : `${this.name}.json`),
           content
           // Do not set to 'utf8'
         )

script/search/search-index-records.js

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+import { fileURLToPath } from 'url'
+import path from 'path'
+import fs from 'fs/promises'
+
+import validateRecords from './validate-records.js'
+import { compress } from '../../lib/search/compress.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export async function writeIndexRecords(
+  name,
+  records,
+  {
+    outDirectory = path.posix.join(__dirname, '../../lib/search/indexes'),
+    compressFiles = true,
+    prettyPrint = false,
+  }
+) {
+  validateRecords(name, records)
+
+  const recordsObject = Object.fromEntries(records.map((record) => [record.objectID, record]))
+  const content = JSON.stringify(recordsObject, undefined, prettyPrint ? 2 : 0)
+
+  const filePath = path.join(
+    outDirectory,
+    compressFiles ? `${name}-records.json.br` : `${name}-records.json`
+  )
+  await fs.writeFile(filePath, compressFiles ? await compress(content) : content)
+
+  return filePath
+}

script/search/sync-search-indices.js

@@ -30,8 +30,13 @@ program
   .option('-d, --dry-run', 'Does not write to disk')
   .option(
     '-o, --out-directory <DIRECTORY>',
-    `Where to dump the created files (default ${DEFAULT_OUT_DIRECTORY}`
+    `Where to dump the created files (default ${DEFAULT_OUT_DIRECTORY})`
   )
+  .option('--no-compression', `Do not Brotli compress the created .json files (default false)`)
+  // Once we've fully removed all Lunr indexing code, we can remove this option
+  // and change where it's used to be that the default is to not generate
+  // any Lunr indexes.
+  .option('--no-lunr-index', `Do not generate a Lunr index, just the records file (default false)`)
   .parse(process.argv)
 
 main(program.opts())
@@ -88,12 +93,18 @@ async function main(opts) {
 
   const outDirectory = opts.outDirectory || DEFAULT_OUT_DIRECTORY
 
+  const compressFiles = !!opts.compression
+
+  const generateLunrIndex = !!opts.lunrIndex
+
   const options = {
     dryRun,
     language,
     notLanguage,
     version,
     outDirectory,
+    compressFiles,
+    generateLunrIndex,
   }
   await searchSync(options)
 }

script/search/sync.js

@@ -1,10 +1,13 @@
 #!/usr/bin/env node
+import chalk from 'chalk'
+
 import languages from '../../lib/languages.js'
 import buildRecords from './build-records.js'
 import findIndexablePages from './find-indexable-pages.js'
 import { allVersions } from '../../lib/all-versions.js'
 import { namePrefix } from '../../lib/search/config.js'
 import LunrIndex from './lunr-search-index.js'
+import { writeIndexRecords } from './search-index-records.js'
 
 // Build a search data file for every combination of product version and language
 // e.g. `github-docs-dotcom-en.json` and `github-docs-2.14-ja.json`
@@ -14,6 +17,8 @@ export default async function syncSearchIndexes({
   dryRun,
   notLanguage,
   outDirectory,
+  compressFiles,
+  generateLunrIndex,
 }) {
   const t0 = new Date()
 
@@ -28,7 +33,9 @@ export default async function syncSearchIndexes({
   )
 
   console.log(
-    `Building indices for ${language || 'all languages'} and ${version || 'all versions'}.\n`
+    `Building indices for ${chalk.yellow(language || 'all languages')} and ${chalk.yellow(
+      version || 'all versions'
+    )}.\n`
   )
 
   // Exclude WIP pages, hidden pages, index pages, etc
@@ -67,11 +74,19 @@ export default async function syncSearchIndexes({
         languageCode,
         redirects
       )
-      const index = new LunrIndex(indexName, records)
+      if (generateLunrIndex) {
+        const index = new LunrIndex(indexName, records)
 
-      if (!dryRun) {
-        await index.write(outDirectory)
-        console.log('wrote index to file: ', indexName)
+        if (!dryRun) {
+          await index.write({ outDirectory, compressFiles })
+          console.log('wrote index to file: ', indexName)
+        }
+      } else {
+        const fileWritten = await writeIndexRecords(indexName, records, {
+          outDirectory,
+          compressFiles,
+        })
+        console.log(`wrote records to ${fileWritten}`)
       }
     }
   }