diff --git a/script/search/lunr-search-index.js b/script/search/lunr-search-index.js index 1599df533959..c770c0bb10f8 100755 --- a/script/search/lunr-search-index.js +++ b/script/search/lunr-search-index.js @@ -78,16 +78,22 @@ export default class LunrIndex { return Object.fromEntries(this.records.map((record) => [record.objectID, record])) } - async write(outDirectory = path.posix.join(__dirname, '../../lib/search/indexes')) { + async write({ + outDirectory = path.posix.join(__dirname, '../../lib/search/indexes'), + compressFiles = true, + }) { this.build() // Write the parsed records await Promise.resolve(this.recordsObject) .then(JSON.stringify) - .then(compress) + .then((str) => (compressFiles ? compress(str) : str)) .then((content) => fs.writeFile( - path.join(outDirectory, `${this.name}-records.json.br`), + path.join( + outDirectory, + compressFiles ? `${this.name}-records.json.br` : `${this.name}-records.json` + ), content // Do not set to 'utf8' ) @@ -96,10 +102,10 @@ export default class LunrIndex { // Write the index await Promise.resolve(this.index) .then(JSON.stringify) - .then(compress) + .then((str) => (compressFiles ? compress(str) : str)) .then((content) => fs.writeFile( - path.join(outDirectory, `${this.name}.json.br`), + path.join(outDirectory, compressFiles ? `${this.name}.json.br` : `${this.name}.json`), content // Do not set to 'utf8' ) diff --git a/script/search/search-index-records.js b/script/search/search-index-records.js new file mode 100644 index 000000000000..85da92e04850 --- /dev/null +++ b/script/search/search-index-records.js @@ -0,0 +1,32 @@ +#!/usr/bin/env node +import { fileURLToPath } from 'url' +import path from 'path' +import fs from 'fs/promises' + +import validateRecords from './validate-records.js' +import { compress } from '../../lib/search/compress.js' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +export async function writeIndexRecords( + name, + records, + { + outDirectory = path.posix.join(__dirname, '../../lib/search/indexes'), + compressFiles = true, + prettyPrint = false, + } +) { + validateRecords(name, records) + + const recordsObject = Object.fromEntries(records.map((record) => [record.objectID, record])) + const content = JSON.stringify(recordsObject, undefined, prettyPrint ? 2 : 0) + + const filePath = path.join( + outDirectory, + compressFiles ? `${name}-records.json.br` : `${name}-records.json` + ) + await fs.writeFile(filePath, compressFiles ? await compress(content) : content) + + return filePath +} diff --git a/script/search/sync-search-indices.js b/script/search/sync-search-indices.js index ab05c90708e1..c314b26e0f8b 100755 --- a/script/search/sync-search-indices.js +++ b/script/search/sync-search-indices.js @@ -30,8 +30,13 @@ program .option('-d, --dry-run', 'Does not write to disk') .option( '-o, --out-directory ', - `Where to dump the created files (default ${DEFAULT_OUT_DIRECTORY}` + `Where to dump the created files (default ${DEFAULT_OUT_DIRECTORY})` ) + .option('--no-compression', `Do not Brotli compress the created .json files (default false)`) + // Once we've fully removed all Lunr indexing code, we can remove this option + // and change where it's used to be that the default is to not generate + // any Lunr indexes. + .option('--no-lunr-index', `Do not generate a Lunr index, just the records file (default false)`) .parse(process.argv) main(program.opts()) @@ -88,12 +93,18 @@ async function main(opts) { const outDirectory = opts.outDirectory || DEFAULT_OUT_DIRECTORY + const compressFiles = !!opts.compression + + const generateLunrIndex = !!opts.lunrIndex + const options = { dryRun, language, notLanguage, version, outDirectory, + compressFiles, + generateLunrIndex, } await searchSync(options) } diff --git a/script/search/sync.js b/script/search/sync.js index 3cc9fb6f289e..fa35ddc527f9 100644 --- a/script/search/sync.js +++ b/script/search/sync.js @@ -1,10 +1,13 @@ #!/usr/bin/env node +import chalk from 'chalk' + import languages from '../../lib/languages.js' import buildRecords from './build-records.js' import findIndexablePages from './find-indexable-pages.js' import { allVersions } from '../../lib/all-versions.js' import { namePrefix } from '../../lib/search/config.js' import LunrIndex from './lunr-search-index.js' +import { writeIndexRecords } from './search-index-records.js' // Build a search data file for every combination of product version and language // e.g. `github-docs-dotcom-en.json` and `github-docs-2.14-ja.json` @@ -14,6 +17,8 @@ export default async function syncSearchIndexes({ dryRun, notLanguage, outDirectory, + compressFiles, + generateLunrIndex, }) { const t0 = new Date() @@ -28,7 +33,9 @@ export default async function syncSearchIndexes({ ) console.log( - `Building indices for ${language || 'all languages'} and ${version || 'all versions'}.\n` + `Building indices for ${chalk.yellow(language || 'all languages')} and ${chalk.yellow( + version || 'all versions' + )}.\n` ) // Exclude WIP pages, hidden pages, index pages, etc @@ -67,11 +74,19 @@ export default async function syncSearchIndexes({ languageCode, redirects ) - const index = new LunrIndex(indexName, records) + if (generateLunrIndex) { + const index = new LunrIndex(indexName, records) - if (!dryRun) { - await index.write(outDirectory) - console.log('wrote index to file: ', indexName) + if (!dryRun) { + await index.write({ outDirectory, compressFiles }) + console.log('wrote index to file: ', indexName) + } + } else { + const fileWritten = await writeIndexRecords(indexName, records, { + outDirectory, + compressFiles, + }) + console.log(`wrote records to ${fileWritten}`) } } }