From 75a7d4abb3575c00c25326298992d26651568ffe Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Thu, 6 Feb 2025 18:23:27 +0000 Subject: [PATCH 01/15] use worker in code to avoid freezing --- .gitignore | 4 +- assets/js/generate-lunr-index.js | 24 +++ assets/js/offline-search.js | 291 ++++++++++++++--------------- assets/js/worker.js | 49 +++++ layouts/docs/baseof.html | 3 + layouts/partials/search-input.html | 5 +- package-lock.json | 13 ++ package.json | 5 +- 8 files changed, 240 insertions(+), 154 deletions(-) create mode 100644 assets/js/generate-lunr-index.js create mode 100644 assets/js/worker.js diff --git a/.gitignore b/.gitignore index 0ac1b8495..b7406f838 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ - public/ resources/ node_modules/ @@ -7,4 +6,5 @@ content/en/docs/latest/ content/static/latest/ content/static/**/*.dtmp content/static/**/*.bkp -content/static/**/*.crswap \ No newline at end of file +content/static/**/*.crswap +assets/json/lunr-index.json \ No newline at end of file diff --git a/assets/js/generate-lunr-index.js b/assets/js/generate-lunr-index.js new file mode 100644 index 000000000..06343d27c --- /dev/null +++ b/assets/js/generate-lunr-index.js @@ -0,0 +1,24 @@ +const fs = require('fs'); +const lunr = require('lunr'); + +const data = JSON.parse(fs.readFileSync('docs/offline-search-index.json')); + +const idx = lunr(function () { + this.ref('ref'); + this.field('title', { boost: 5 }); + this.field('categories', { boost: 3 }); + this.field('tags', { boost: 3 }); + this.field('description', { boost: 2 }); + this.field('body'); + + data.forEach((doc) => {let docToAdd; + if (doc + && doc.ref !== undefined + && !doc.ref.includes('/_shared/') + ) { + this.add(doc); + } + }); +}); + +fs.writeFileSync('assets/json/lunr-index.json', JSON.stringify(idx)); \ No newline at end of file diff --git a/assets/js/offline-search.js b/assets/js/offline-search.js index 6f99da65a..3bd1602ed 100644 --- a/assets/js/offline-search.js +++ b/assets/js/offline-search.js @@ -17,192 +17,185 @@ '' ); - // - // Register handler - // - - $searchInput.on('change', (event) => { - render($(event.target)); - - // Hide keyboard on mobile browser - $searchInput.blur(); - }); - - // Prevent reloading page by enter key on sidebar search. - $searchInput.closest('form').on('submit', () => { - return false; - }); - // // Lunr // let idx = null; // Lunr index const resultDetails = new Map(); // Will hold the data for the search results (titles and summaries) + let worker = null; - // Set up for an Ajax call to request the JSON data file that is created by Hugo's build process + + if (window.Worker) { + worker = new Worker('/js/worker.js'); + const url = '/json/lunr-index.json'; + + worker.postMessage({ type: 'init', url: url }); + + worker.onerror = function (error) { + console.error('Error in worker:', error); + }; + } + $.ajax($searchInput.data('offline-search-index-json-src')).then( (data) => { - idx = lunr(function () { - this.ref('ref'); - - // If you added more searchable fields to the search index, list them here. - // Here you can specify searchable fields to the search index - e.g. individual toxonomies for you project - // With "boost" you can add weighting for specific (default weighting without boost: 1) - this.field('title', { boost: 5 }); - this.field('categories', { boost: 3 }); - this.field('tags', { boost: 3 }); - this.field('description', { boost: 2 }); - this.field('body'); - - const searchPath = $searchInput.data('search-path'); - - data.forEach((doc) => { - let docToAdd; - if (searchPath !== undefined && doc.ref.startsWith(searchPath)) { - docToAdd = doc; - } else if (searchPath === undefined) { - docToAdd = doc; - } - - if (docToAdd - && docToAdd.ref !== undefined - && !docToAdd.ref.includes('/_shared/') - ) { - this.add(doc); - - resultDetails.set(doc.ref, { - title: doc.title, - excerpt: doc.excerpt, - }); - } + data.forEach((doc) => { + resultDetails.set(doc.ref, { + version: doc.version, + title: doc.title, + excerpt: doc.excerpt, }); }); - - $searchInput.trigger('change'); } ); + let currentTarget = null; + + worker.onmessage = function (event) { + if (event.data.type === 'search') { + const results = event.data.results + console.log('Search results:', results); + const $html = $('
'); + + $html.append( + $('
') + .css({ + display: 'flex', + justifyContent: 'space-between', + marginBottom: '1em', + }) + .append( + $('') + .text('Search results') + .css({ fontWeight: 'bold' }) + ) + .append( + $('') + .addClass('fas fa-times search-result-close-button') + .css({ + cursor: 'pointer', + }) + ) + ); + + const $searchResultBody = $('
').css({ + maxHeight: `calc(100vh - ${currentTarget.offset().top - + $(window).scrollTop() + + 180 + }px)`, + overflowY: 'auto', + }); + $html.append($searchResultBody); + + if (results.length === 0) { + currentTarget.append( + $('

').text(`No results found for query "${searchQuery}"`) + ); + } else { + results.forEach((r) => { + const doc = resultDetails.get(r.ref); + + const href = + $searchInput.data('offline-search-base-href') + + r.ref.replace(/^\//, ''); + + const $entry = $('

').addClass('mt-4').addClass('search-result'); + + $entry.append( + $('') + .addClass('d-block') + .css({ + fontSize: '1.2rem', + }) + .attr('href', href) + .text(doc.title) + ); + + $entry.append( + $('').addClass('d-block text-muted').text(r.ref) + ); + + $entry.append($('

').text(doc.excerpt)); + + $searchResultBody.append($entry); + }); + } + + currentTarget.on('shown.bs.popover', () => { + $('.search-result-close-button').on('click', () => { + currentTarget.val(''); + currentTarget.trigger('change'); + }); + }); + + currentTarget + .data('content', $html[0].outerHTML) + .popover('show'); + } + } + const render = ($targetSearchInput) => { // Dispose the previous result $targetSearchInput.popover('dispose'); + currentTarget = $targetSearchInput; // // Search // - if (idx === null) { - return; - } const searchQuery = $targetSearchInput.val(); if (searchQuery === '') { return; } - const results = idx - .query((q) => { - const tokens = lunr.tokenizer(searchQuery.toLowerCase()); - tokens.forEach((token) => { - const queryString = token.toString(); - q.term(queryString, { - boost: 100, - }); - q.term(queryString, { - wildcard: - lunr.Query.wildcard.LEADING | - lunr.Query.wildcard.TRAILING, - boost: 10, - }); - q.term(queryString, { - editDistance: 2, - }); - }); - }) - .slice( - 0, - $targetSearchInput.data('offline-search-max-results') - ); + worker.postMessage({ type: 'search', query: searchQuery, maxResults: $targetSearchInput.data('offline-search-max-results') }); + + // const results = idx + // .query((q) => { + // const tokens = lunr.tokenizer(searchQuery.toLowerCase()); + // tokens.forEach((token) => { + // const queryString = token.toString(); + // q.term(queryString, { + // boost: 100, + // }); + // q.term(queryString, { + // wildcard: + // lunr.Query.wildcard.LEADING | + // lunr.Query.wildcard.TRAILING, + // boost: 10, + // }); + // q.term(queryString, { + // editDistance: 2, + // }); + // }); + // }) + // .slice( + // 0, + // $targetSearchInput.data('offline-search-max-results') + // ); // // Make result html // - const $html = $('

'); - - $html.append( - $('
') - .css({ - display: 'flex', - justifyContent: 'space-between', - marginBottom: '1em', - }) - .append( - $('') - .text('Search results') - .css({ fontWeight: 'bold' }) - ) - .append( - $('') - .addClass('fas fa-times search-result-close-button') - .css({ - cursor: 'pointer', - }) - ) - ); - - const $searchResultBody = $('
').css({ - maxHeight: `calc(100vh - ${$targetSearchInput.offset().top - - $(window).scrollTop() + - 180 - }px)`, - overflowY: 'auto', - }); - $html.append($searchResultBody); - - if (results.length === 0) { - $searchResultBody.append( - $('

').text(`No results found for query "${searchQuery}"`) - ); - } else { - results.forEach((r) => { - const doc = resultDetails.get(r.ref); - const href = - $searchInput.data('offline-search-base-href') + - r.ref.replace(/^\//, ''); - - const $entry = $('

').addClass('mt-4').addClass('search-result'); - - $entry.append( - $('') - .addClass('d-block') - .css({ - fontSize: '1.2rem', - }) - .attr('href', href) - .text(doc.title) - ); - $entry.append( - $('').addClass('d-block text-muted').text(r.ref) - ); + }; - $entry.append($('

').text(doc.excerpt)); + // + // Register handler + // - $searchResultBody.append($entry); - }); - } + $searchInput.on('change', (event) => { + render($(event.target)); - $targetSearchInput.on('shown.bs.popover', () => { - $('.search-result-close-button').on('click', () => { - $targetSearchInput.val(''); - $targetSearchInput.trigger('change'); - }); - }); + // Hide keyboard on mobile browser + $searchInput.blur(); + }); - $targetSearchInput - .data('content', $html[0].outerHTML) - .popover('show'); - }; + // Prevent reloading page by enter key on sidebar search. + $searchInput.closest('form').on('submit', () => { + return false; + }); }); })(jQuery); diff --git a/assets/js/worker.js b/assets/js/worker.js new file mode 100644 index 000000000..5c2b6a416 --- /dev/null +++ b/assets/js/worker.js @@ -0,0 +1,49 @@ +importScripts('https://unpkg.com/lunr@2.3.8/lunr.min.js'); + +let idx; +let indexReadyPromise; + +// Initialize the index +self.onmessage = async function(event) { + if (event.data.type === 'init') { + indexReadyPromise = new Promise(async (resolve, reject) => { + try { + console.log(event.data); + const response = await fetch(event.data.url); + const data = await response.json(); + idx = lunr.Index.load(data); + resolve(); + self.postMessage({ type: 'init', status: 'success' }); + } catch (error) { + reject(error); + self.postMessage({ type: 'init', status: 'error', message: error.message }); + } + }); + } else if (event.data.type === 'search') { + try { + await indexReadyPromise; + const results = idx + .query((q) => { + const tokens = lunr.tokenizer(event.data.query.toLowerCase()); + tokens.forEach((token) => { + const queryString = token.toString(); + q.term(queryString, { + boost: 100, + }); + q.term(queryString, { + wildcard: lunr.Query.wildcard.LEADING | lunr.Query.wildcard.TRAILING, + boost: 10, + }); + q.term(queryString, { + editDistance: 2, + }); + }); + }) + .slice(0, event.data.maxResults); + + self.postMessage({ type: 'search', results: results }); + } catch (error) { + self.postMessage({ type: 'search', status: 'error', message: 'Index not ready' }); + } + } +}; \ No newline at end of file diff --git a/layouts/docs/baseof.html b/layouts/docs/baseof.html index 64edd70ef..9c02884fc 100644 --- a/layouts/docs/baseof.html +++ b/layouts/docs/baseof.html @@ -26,5 +26,8 @@ {{ partial "footer.html" . }}

{{ partial "scripts.html" . }} + {{ block "main" . }}{{ end }} + {{ $script := resources.Get "js/worker.js"}} + \ No newline at end of file diff --git a/layouts/partials/search-input.html b/layouts/partials/search-input.html index bf974fcc6..93712b74d 100644 --- a/layouts/partials/search-input.html +++ b/layouts/partials/search-input.html @@ -1,3 +1,4 @@ +{{- $lunrIndex := resources.Get "json/lunr-index.json" }} {{ if or .Site.Params.gcs_engine_id .Site.Params.algolia_docsearch }} {{ else if .Site.Params.offlineSearch }} @@ -6,8 +7,7 @@ {{ $path = index (findRE `^\/[^\/]*\/` .Page.RelPermalink 1) 0}} {{- end }} {{- $offlineSearchIndex := resources.Get "json/offline-search-index.json" | resources.ExecuteAsTemplate "offline-search-index.json" . }} - {{- /* Use `md5` as finger print hash function to shorten file name to avoid `file name too long` error. */}} - {{ $offlineSearchIndexFingerprint := $offlineSearchIndex | resources.Fingerprint "md5" }} + {{ $offlineSearchIndexFingerprint := $offlineSearchIndex }} diff --git a/package-lock.json b/package-lock.json index ff5a3fa56..75e81066e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "tech-doc-hugo", "version": "0.0.1", "license": "ISC", + "dependencies": { + "lunr": "2.3.8" + }, "devDependencies": { "autoprefixer": "^9.8.6", "postcss-cli": "^7.1.2" @@ -724,6 +727,11 @@ "node": ">=4" } }, + "node_modules/lunr": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.8.tgz", + "integrity": "sha512-oxMeX/Y35PNFuZoHp+jUj5OSEmLCaIH4KTFJh7a93cHBoFmpw2IoPs22VIz7vyO2YUnx2Tn9dzIwO2P/4quIRg==" + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -1822,6 +1830,11 @@ "chalk": "^2.0.1" } }, + "lunr": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.8.tgz", + "integrity": "sha512-oxMeX/Y35PNFuZoHp+jUj5OSEmLCaIH4KTFJh7a93cHBoFmpw2IoPs22VIz7vyO2YUnx2Tn9dzIwO2P/4quIRg==" + }, "merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", diff --git a/package.json b/package.json index f11e053ed..7aa76fb9b 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Hugo theme for technical documentation.", "main": "none.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "build": "hugo && node ./assets/js/generate-lunr-index.js" }, "repository": { "type": "git", @@ -19,5 +19,8 @@ "devDependencies": { "autoprefixer": "^9.8.6", "postcss-cli": "^7.1.2" + }, + "dependencies": { + "lunr": "2.3.8" } } From c182e9f9877768f38cf66e59f1b0b4eae590d1ae Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 10:44:44 +0000 Subject: [PATCH 02/15] make search reliable --- assets/js/offline-search.js | 63 ++++++++----------------------------- assets/js/worker.js | 31 ++++++++++++++---- 2 files changed, 38 insertions(+), 56 deletions(-) diff --git a/assets/js/offline-search.js b/assets/js/offline-search.js index 3bd1602ed..7ae91eb81 100644 --- a/assets/js/offline-search.js +++ b/assets/js/offline-search.js @@ -30,31 +30,22 @@ worker = new Worker('/js/worker.js'); const url = '/json/lunr-index.json'; - worker.postMessage({ type: 'init', url: url }); + worker.postMessage({ + type: 'init', + lunrIndexUrl: url, + rawIndexUrl: $searchInput.data('offline-search-index-json-src') + }); worker.onerror = function (error) { console.error('Error in worker:', error); }; } - - $.ajax($searchInput.data('offline-search-index-json-src')).then( - (data) => { - data.forEach((doc) => { - resultDetails.set(doc.ref, { - version: doc.version, - title: doc.title, - excerpt: doc.excerpt, - }); - }); - } - ); let currentTarget = null; worker.onmessage = function (event) { if (event.data.type === 'search') { - const results = event.data.results - console.log('Search results:', results); + const docs = event.data.docs; const $html = $('
'); $html.append( @@ -87,17 +78,19 @@ }); $html.append($searchResultBody); - if (results.length === 0) { + if (docs.size === 0) { currentTarget.append( $('

').text(`No results found for query "${searchQuery}"`) ); } else { - results.forEach((r) => { - const doc = resultDetails.get(r.ref); + docs.forEach((doc, key) => { + if (doc === undefined) { + return; + } const href = $searchInput.data('offline-search-base-href') + - r.ref.replace(/^\//, ''); + key.replace(/^\//, ''); const $entry = $('

').addClass('mt-4').addClass('search-result'); @@ -112,7 +105,7 @@ ); $entry.append( - $('').addClass('d-block text-muted').text(r.ref) + $('').addClass('d-block text-muted').text(key) ); $entry.append($('

').text(doc.excerpt)); @@ -150,36 +143,6 @@ } worker.postMessage({ type: 'search', query: searchQuery, maxResults: $targetSearchInput.data('offline-search-max-results') }); - - // const results = idx - // .query((q) => { - // const tokens = lunr.tokenizer(searchQuery.toLowerCase()); - // tokens.forEach((token) => { - // const queryString = token.toString(); - // q.term(queryString, { - // boost: 100, - // }); - // q.term(queryString, { - // wildcard: - // lunr.Query.wildcard.LEADING | - // lunr.Query.wildcard.TRAILING, - // boost: 10, - // }); - // q.term(queryString, { - // editDistance: 2, - // }); - // }); - // }) - // .slice( - // 0, - // $targetSearchInput.data('offline-search-max-results') - // ); - - // - // Make result html - // - - }; // diff --git a/assets/js/worker.js b/assets/js/worker.js index 5c2b6a416..36656d492 100644 --- a/assets/js/worker.js +++ b/assets/js/worker.js @@ -1,17 +1,27 @@ importScripts('https://unpkg.com/lunr@2.3.8/lunr.min.js'); let idx; +const resultDetails = new Map(); // Will hold the data for the search results (titles and summaries) let indexReadyPromise; // Initialize the index -self.onmessage = async function(event) { +self.onmessage = async function (event) { if (event.data.type === 'init') { indexReadyPromise = new Promise(async (resolve, reject) => { try { - console.log(event.data); - const response = await fetch(event.data.url); - const data = await response.json(); - idx = lunr.Index.load(data); + const rawIndex = await fetch(event.data.rawIndexUrl); + let json = await rawIndex.json(); + json.forEach((doc) => { + resultDetails.set(doc.ref, { + version: doc.version, + title: doc.title, + excerpt: doc.excerpt, + }); + }); + + const lunrIndex = await fetch(event.data.lunrIndexUrl); + json = await lunrIndex.json(); + idx = lunr.Index.load(json); resolve(); self.postMessage({ type: 'init', status: 'success' }); } catch (error) { @@ -41,7 +51,16 @@ self.onmessage = async function(event) { }) .slice(0, event.data.maxResults); - self.postMessage({ type: 'search', results: results }); + const docs = new Map(); + results.forEach((result) => { + if (resultDetails.get(result.ref) === undefined) { + return; + } + + docs.set(result.ref, resultDetails.get(result.ref)); + }); + + self.postMessage({ type: 'search', status: 'success', docs: docs }); } catch (error) { self.postMessage({ type: 'search', status: 'error', message: 'Index not ready' }); } From fb2e8bb2f0376f7b23f2d988ef5e5cd6fedf53ac Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 11:06:15 +0000 Subject: [PATCH 03/15] added index creation to pipelines --- .github/workflows/checklink.yml | 3 +++ .github/workflows/hugo.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/checklink.yml b/.github/workflows/checklink.yml index d95853621..7a297f79b 100644 --- a/.github/workflows/checklink.yml +++ b/.github/workflows/checklink.yml @@ -37,6 +37,9 @@ jobs: HUGO_ENVIRONMENT: production HUGO_ENV: production run: hugo --environment GitHubPages -d $GITHUB_WORKSPACE/dist + - name: Generate Search index + run: | + node .\assets\js\generate-lunr-index.js - name: Test HTML uses: wjdp/htmltest-action@master with: diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index b62b3cdea..d2c15714a 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -63,6 +63,9 @@ jobs: HUGO_ENV: production run: | hugo --environment GitHubPages + - name: Generate Search index + run: | + node .\assets\js\generate-lunr-index.js - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: From fe9e5ee777bdc1df0e00a2a3f81a2cdf536619ae Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 11:46:14 +0000 Subject: [PATCH 04/15] make worker only load lunr when called --- assets/js/worker.js | 8 +++++--- layouts/docs/baseof.html | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/assets/js/worker.js b/assets/js/worker.js index 36656d492..83bf5ddb2 100644 --- a/assets/js/worker.js +++ b/assets/js/worker.js @@ -1,4 +1,6 @@ -importScripts('https://unpkg.com/lunr@2.3.8/lunr.min.js'); +if (typeof importScripts === 'function') { + importScripts('https://unpkg.com/lunr@2.3.8/lunr.min.js'); +} let idx; const resultDetails = new Map(); // Will hold the data for the search results (titles and summaries) @@ -53,10 +55,10 @@ self.onmessage = async function (event) { const docs = new Map(); results.forEach((result) => { - if (resultDetails.get(result.ref) === undefined) { + if (resultDetails.get(result.ref) === undefined) { return; } - + docs.set(result.ref, resultDetails.get(result.ref)); }); diff --git a/layouts/docs/baseof.html b/layouts/docs/baseof.html index 9c02884fc..c8e57e24a 100644 --- a/layouts/docs/baseof.html +++ b/layouts/docs/baseof.html @@ -26,7 +26,6 @@ {{ partial "footer.html" . }}

{{ partial "scripts.html" . }} - {{ block "main" . }}{{ end }} {{ $script := resources.Get "js/worker.js"}} From 1ff154eb327ecc10ac14452c943c46fec1328267 Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 11:57:48 +0000 Subject: [PATCH 05/15] add initial lunr index --- assets/json/lunr-index.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 assets/json/lunr-index.json diff --git a/assets/json/lunr-index.json b/assets/json/lunr-index.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/assets/json/lunr-index.json @@ -0,0 +1 @@ +{} \ No newline at end of file From a12583ce87e40f1eef34e97beb8d49edff6bb6e8 Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 12:15:56 +0000 Subject: [PATCH 06/15] fix pipelines --- .github/workflows/checklink.yml | 2 +- .github/workflows/hugo.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checklink.yml b/.github/workflows/checklink.yml index c3b6b4bbe..d89cde544 100644 --- a/.github/workflows/checklink.yml +++ b/.github/workflows/checklink.yml @@ -39,7 +39,7 @@ jobs: run: hugo --environment GitHubPages -d $GITHUB_WORKSPACE/dist --buildFuture - name: Generate Search index run: | - node .\assets\js\generate-lunr-index.js + node ./assets/js/generate-lunr-index.js - name: Test HTML uses: wjdp/htmltest-action@master with: diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index d2c15714a..b8e4d2a34 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -65,7 +65,7 @@ jobs: hugo --environment GitHubPages - name: Generate Search index run: | - node .\assets\js\generate-lunr-index.js + node ./assets/js/generate-lunr-index.js - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: From 5d69ee83c9a301523a1634032a7ad6d2f96cb63b Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 12:27:35 +0000 Subject: [PATCH 07/15] Remove ignored file from tracking --- assets/json/lunr-index.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 assets/json/lunr-index.json diff --git a/assets/json/lunr-index.json b/assets/json/lunr-index.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/assets/json/lunr-index.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file From a84c6d662be64fc18b9539ac382a66beaca056d5 Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 12:30:28 +0000 Subject: [PATCH 08/15] add empty index --- assets/json/lunr-index.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 assets/json/lunr-index.json diff --git a/assets/json/lunr-index.json b/assets/json/lunr-index.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/assets/json/lunr-index.json @@ -0,0 +1 @@ +{} \ No newline at end of file From ae680c8444a51afe06fd7008fb6b254b3919bfe0 Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 13:11:10 +0000 Subject: [PATCH 09/15] fix paths --- assets/js/generate-lunr-index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/js/generate-lunr-index.js b/assets/js/generate-lunr-index.js index 06343d27c..b10c9bb61 100644 --- a/assets/js/generate-lunr-index.js +++ b/assets/js/generate-lunr-index.js @@ -1,7 +1,7 @@ const fs = require('fs'); const lunr = require('lunr'); -const data = JSON.parse(fs.readFileSync('docs/offline-search-index.json')); +const data = JSON.parse(fs.readFileSync('./docs/offline-search-index.json')); const idx = lunr(function () { this.ref('ref'); @@ -21,4 +21,4 @@ const idx = lunr(function () { }); }); -fs.writeFileSync('assets/json/lunr-index.json', JSON.stringify(idx)); \ No newline at end of file +fs.writeFileSync('./assets/json/lunr-index.json', JSON.stringify(idx)); \ No newline at end of file From d7484015669e5e443ca91f3965bf525c10d04e65 Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 13:27:36 +0000 Subject: [PATCH 10/15] update generate lunr index --- .github/workflows/checklink.yml | 2 +- assets/js/generate-lunr-index.js | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checklink.yml b/.github/workflows/checklink.yml index d89cde544..54001eb1d 100644 --- a/.github/workflows/checklink.yml +++ b/.github/workflows/checklink.yml @@ -39,7 +39,7 @@ jobs: run: hugo --environment GitHubPages -d $GITHUB_WORKSPACE/dist --buildFuture - name: Generate Search index run: | - node ./assets/js/generate-lunr-index.js + node ./assets/js/generate-lunr-index.js $GITHUB_WORKSPACE/dist - name: Test HTML uses: wjdp/htmltest-action@master with: diff --git a/assets/js/generate-lunr-index.js b/assets/js/generate-lunr-index.js index b10c9bb61..3ae12c598 100644 --- a/assets/js/generate-lunr-index.js +++ b/assets/js/generate-lunr-index.js @@ -1,7 +1,10 @@ const fs = require('fs'); const lunr = require('lunr'); -const data = JSON.parse(fs.readFileSync('./docs/offline-search-index.json')); +const args = process.argv.slice(2); +const destination = args[0] ?? "."; + +const data = JSON.parse(fs.readFileSync(`${destination}/docs/offline-search-index.json`)); const idx = lunr(function () { this.ref('ref'); @@ -21,4 +24,10 @@ const idx = lunr(function () { }); }); -fs.writeFileSync('./assets/json/lunr-index.json', JSON.stringify(idx)); \ No newline at end of file +fs.writeFileSync(`${destination}/assets/json/lunr-index.json`, JSON.stringify(idx)); + +// check if file got created +if (!fs.existsSync(`${destination}/assets/json/lunr-index.json`)) { + console.error('Failed to create lunr index'); + process.exit(1); +} \ No newline at end of file From 5e6798aa8b49061b555ca09cf2561c49df09dae3 Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 16:33:05 +0000 Subject: [PATCH 11/15] Remove ignored file from tracking --- assets/js/generate-lunr-index.js | 12 +++++++----- assets/js/offline-search.js | 17 +++++++++++------ assets/js/worker.js | 18 +++++++++++++----- assets/json/lunr-index.json | 1 - 4 files changed, 31 insertions(+), 17 deletions(-) delete mode 100644 assets/json/lunr-index.json diff --git a/assets/js/generate-lunr-index.js b/assets/js/generate-lunr-index.js index 3ae12c598..cd33edcc4 100644 --- a/assets/js/generate-lunr-index.js +++ b/assets/js/generate-lunr-index.js @@ -2,9 +2,11 @@ const fs = require('fs'); const lunr = require('lunr'); const args = process.argv.slice(2); -const destination = args[0] ?? "."; +const destination = args[0] === undefined + ? "./docs" + : args[0]; -const data = JSON.parse(fs.readFileSync(`${destination}/docs/offline-search-index.json`)); +const data = JSON.parse(fs.readFileSync(`${destination}/offline-search-index.json`)); const idx = lunr(function () { this.ref('ref'); @@ -14,7 +16,7 @@ const idx = lunr(function () { this.field('description', { boost: 2 }); this.field('body'); - data.forEach((doc) => {let docToAdd; + data.forEach((doc) => { if (doc && doc.ref !== undefined && !doc.ref.includes('/_shared/') @@ -24,10 +26,10 @@ const idx = lunr(function () { }); }); -fs.writeFileSync(`${destination}/assets/json/lunr-index.json`, JSON.stringify(idx)); +fs.writeFileSync(`${destination}/lunr-index.json`, JSON.stringify(idx)); // check if file got created -if (!fs.existsSync(`${destination}/assets/json/lunr-index.json`)) { +if (!fs.existsSync(`${destination}/lunr-index.json`)) { console.error('Failed to create lunr index'); process.exit(1); } \ No newline at end of file diff --git a/assets/js/offline-search.js b/assets/js/offline-search.js index 7ae91eb81..f7bd7e648 100644 --- a/assets/js/offline-search.js +++ b/assets/js/offline-search.js @@ -28,12 +28,13 @@ if (window.Worker) { worker = new Worker('/js/worker.js'); - const url = '/json/lunr-index.json'; + const url = '/lunr-index.json'; - worker.postMessage({ - type: 'init', - lunrIndexUrl: url, - rawIndexUrl: $searchInput.data('offline-search-index-json-src') + worker.postMessage({ + type: 'init', + currentPath: window.location.pathname, + lunrIndexUrl: url, + rawIndexUrl: $searchInput.data('offline-search-index-json-src') }); worker.onerror = function (error) { @@ -142,7 +143,11 @@ return; } - worker.postMessage({ type: 'search', query: searchQuery, maxResults: $targetSearchInput.data('offline-search-max-results') }); + worker.postMessage({ + type: 'search', + query: searchQuery, + maxResults: $targetSearchInput.data('offline-search-max-results') + }); }; // diff --git a/assets/js/worker.js b/assets/js/worker.js index 83bf5ddb2..a81131d99 100644 --- a/assets/js/worker.js +++ b/assets/js/worker.js @@ -3,22 +3,29 @@ if (typeof importScripts === 'function') { } let idx; +const versionRegex = new RegExp("^\/docs\/([0-9\.]*|latest)\/"); const resultDetails = new Map(); // Will hold the data for the search results (titles and summaries) let indexReadyPromise; // Initialize the index self.onmessage = async function (event) { if (event.data.type === 'init') { + const regexResults = versionRegex.exec(event.data.currentPath); + const version = regexResults + ? regexResults[1] + : undefined; indexReadyPromise = new Promise(async (resolve, reject) => { try { const rawIndex = await fetch(event.data.rawIndexUrl); let json = await rawIndex.json(); json.forEach((doc) => { - resultDetails.set(doc.ref, { - version: doc.version, - title: doc.title, - excerpt: doc.excerpt, - }); + if (version === undefined || doc.ref.startsWith(version)) { + resultDetails.set(doc.ref, { + version: doc.version, + title: doc.title, + excerpt: doc.excerpt, + }); + } }); const lunrIndex = await fetch(event.data.lunrIndexUrl); @@ -51,6 +58,7 @@ self.onmessage = async function (event) { }); }); }) + .filter((result) => resultDetails.has(result.ref)) .slice(0, event.data.maxResults); const docs = new Map(); diff --git a/assets/json/lunr-index.json b/assets/json/lunr-index.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/assets/json/lunr-index.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file From 8eea3d7072729d25aeb153ab451da67ded1c8a7f Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Mon, 10 Feb 2025 16:33:41 +0000 Subject: [PATCH 12/15] remove json from search input --- layouts/partials/search-input.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/layouts/partials/search-input.html b/layouts/partials/search-input.html index 93712b74d..733714315 100644 --- a/layouts/partials/search-input.html +++ b/layouts/partials/search-input.html @@ -1,4 +1,3 @@ -{{- $lunrIndex := resources.Get "json/lunr-index.json" }} {{ if or .Site.Params.gcs_engine_id .Site.Params.algolia_docsearch }} {{ else if .Site.Params.offlineSearch }} @@ -23,7 +22,6 @@ */}} data-search-path="{{ $path }}" data-offline-search-index-json-src="{{ $offlineSearchIndexFingerprint.RelPermalink }}" - data-lunr-index-json-src="{{ $lunrIndex.RelPermalink }}" data-offline-search-base-href="/" data-offline-search-max-results="{{ .Site.Params.offlineSearchMaxResults | default 10 }}" > From aecfe6c0d5e57690b6ea0fe83eb026c6658931dd Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Tue, 11 Feb 2025 10:28:52 +0000 Subject: [PATCH 13/15] update mechanism --- .gitignore | 2 +- assets/js/generate-lunr-index.js | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d0f33257e..4c46e5463 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ content/static/latest/ content/static/**/*.dtmp content/static/**/*.bkp content/static/**/*.crswap -assets/json/lunr-index.json \ No newline at end of file +content/static/lunr-index.json \ No newline at end of file diff --git a/assets/js/generate-lunr-index.js b/assets/js/generate-lunr-index.js index cd33edcc4..9851a0d0a 100644 --- a/assets/js/generate-lunr-index.js +++ b/assets/js/generate-lunr-index.js @@ -2,11 +2,19 @@ const fs = require('fs'); const lunr = require('lunr'); const args = process.argv.slice(2); -const destination = args[0] === undefined - ? "./docs" - : args[0]; -const data = JSON.parse(fs.readFileSync(`${destination}/offline-search-index.json`)); +// Arguments should only be provided from a pipeline build. +const isFromPipeline = args[0] !== undefined; + +const source = isFromPipeline + ? args[0] + : "./docs"; + +const destination = isFromPipeline + ? `${args[0]}` + : "./docs"; + +const data = JSON.parse(fs.readFileSync(`${source}/offline-search-index.json`)); const idx = lunr(function () { this.ref('ref'); @@ -26,10 +34,14 @@ const idx = lunr(function () { }); }); +if (!isFromPipeline) { + fs.writeFileSync(`./content/static/lunr-index.json`, JSON.stringify(idx)); +} + fs.writeFileSync(`${destination}/lunr-index.json`, JSON.stringify(idx)); // check if file got created if (!fs.existsSync(`${destination}/lunr-index.json`)) { - console.error('Failed to create lunr index'); + console.error('Failed to create lunr index, hugo must be build using `hugo` command before running this script.'); process.exit(1); } \ No newline at end of file From 5b755e793451edceb1be8a447f1391947a3cf5e8 Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Tue, 11 Feb 2025 13:07:48 +0000 Subject: [PATCH 14/15] make it work properly with versions --- assets/js/offline-search.js | 8 +++--- assets/js/worker.js | 49 +++++++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/assets/js/offline-search.js b/assets/js/offline-search.js index f7bd7e648..30c7e699b 100644 --- a/assets/js/offline-search.js +++ b/assets/js/offline-search.js @@ -32,7 +32,6 @@ worker.postMessage({ type: 'init', - currentPath: window.location.pathname, lunrIndexUrl: url, rawIndexUrl: $searchInput.data('offline-search-index-json-src') }); @@ -79,9 +78,9 @@ }); $html.append($searchResultBody); - if (docs.size === 0) { - currentTarget.append( - $('

').text(`No results found for query "${searchQuery}"`) + if (docs === undefined || docs.size === 0) { + $searchResultBody.append( + $('

').text(`No results found for query "${event.data.query}"`) ); } else { docs.forEach((doc, key) => { @@ -145,6 +144,7 @@ worker.postMessage({ type: 'search', + currentPath: window.location.pathname, query: searchQuery, maxResults: $targetSearchInput.data('offline-search-max-results') }); diff --git a/assets/js/worker.js b/assets/js/worker.js index a81131d99..7af140584 100644 --- a/assets/js/worker.js +++ b/assets/js/worker.js @@ -10,22 +10,16 @@ let indexReadyPromise; // Initialize the index self.onmessage = async function (event) { if (event.data.type === 'init') { - const regexResults = versionRegex.exec(event.data.currentPath); - const version = regexResults - ? regexResults[1] - : undefined; indexReadyPromise = new Promise(async (resolve, reject) => { try { const rawIndex = await fetch(event.data.rawIndexUrl); let json = await rawIndex.json(); json.forEach((doc) => { - if (version === undefined || doc.ref.startsWith(version)) { - resultDetails.set(doc.ref, { - version: doc.version, - title: doc.title, - excerpt: doc.excerpt, - }); - } + resultDetails.set(doc.ref, { + version: doc.version, + title: doc.title, + excerpt: doc.excerpt, + }); }); const lunrIndex = await fetch(event.data.lunrIndexUrl); @@ -41,6 +35,11 @@ self.onmessage = async function (event) { } else if (event.data.type === 'search') { try { await indexReadyPromise; + const regexResults = versionRegex.exec(event.data.currentPath); + const version = regexResults + ? regexResults[0] + : undefined; + const results = idx .query((q) => { const tokens = lunr.tokenizer(event.data.query.toLowerCase()); @@ -57,20 +56,32 @@ self.onmessage = async function (event) { editDistance: 2, }); }); - }) - .filter((result) => resultDetails.has(result.ref)) - .slice(0, event.data.maxResults); + }); const docs = new Map(); - results.forEach((result) => { + let count = 0; + + for (const result of results) { + if (count >= event.data.maxResults) { + break; + } + if (resultDetails.get(result.ref) === undefined) { - return; + continue; } - docs.set(result.ref, resultDetails.get(result.ref)); - }); + if (version === undefined) { + docs.set(result.ref, resultDetails.get(result.ref)); + } else if (result.ref.startsWith(version)) { + docs.set(result.ref, resultDetails.get(result.ref)); + } else { + continue; + } + + count++; + } - self.postMessage({ type: 'search', status: 'success', docs: docs }); + self.postMessage({ type: 'search', status: 'success', query: event.data.query , docs: docs }); } catch (error) { self.postMessage({ type: 'search', status: 'error', message: 'Index not ready' }); } From 6c610b09a5e43247601aa705d15cc622a608100e Mon Sep 17 00:00:00 2001 From: cortex-lp Date: Tue, 11 Feb 2025 14:02:27 +0000 Subject: [PATCH 15/15] add summaries to js code --- assets/js/generate-lunr-index.js | 7 +++++++ assets/js/offline-search.js | 30 ++++++++---------------------- assets/js/worker.js | 14 ++++++++++++-- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/assets/js/generate-lunr-index.js b/assets/js/generate-lunr-index.js index 9851a0d0a..51519fb9e 100644 --- a/assets/js/generate-lunr-index.js +++ b/assets/js/generate-lunr-index.js @@ -1,6 +1,13 @@ const fs = require('fs'); const lunr = require('lunr'); +/** + * This script is used to generate a lunr index from the offline-search-index.json file. + * Hugo must be built before running this script, as jt requires the offline-search-index.json file to have been generated. + * + * The script will output a lunr-index.json file in the content/static directory and the docs directory. + */ + const args = process.argv.slice(2); // Arguments should only be provided from a pipeline build. diff --git a/assets/js/offline-search.js b/assets/js/offline-search.js index 30c7e699b..7151cf686 100644 --- a/assets/js/offline-search.js +++ b/assets/js/offline-search.js @@ -1,4 +1,10 @@ -// Adapted from code by Matt Walters https://www.mattwalters.net/posts/hugo-and-lunr/ + +/** + * This script is used for the offline search feature. + * It calls the worker.js to generate the index and search the index. + * + * Adapted from code by Matt Walters https://www.mattwalters.net/posts/hugo-and-lunr/ + */ (function ($) { 'use strict'; @@ -6,10 +12,6 @@ $(document).ready(function () { const $searchInput = $('.td-search-input'); - // - // Options for popover - // - $searchInput.data('html', true); $searchInput.data('placement', 'bottom'); $searchInput.data( @@ -17,15 +19,8 @@ '

' ); - // - // Lunr - // - - let idx = null; // Lunr index - const resultDetails = new Map(); // Will hold the data for the search results (titles and summaries) let worker = null; - if (window.Worker) { worker = new Worker('/js/worker.js'); const url = '/lunr-index.json'; @@ -128,15 +123,9 @@ } const render = ($targetSearchInput) => { - // Dispose the previous result $targetSearchInput.popover('dispose'); currentTarget = $targetSearchInput; - // - // Search - // - - const searchQuery = $targetSearchInput.val(); if (searchQuery === '') { return; @@ -150,10 +139,7 @@ }); }; - // - // Register handler - // - + // Renders the search results when the input changes. $searchInput.on('change', (event) => { render($(event.target)); diff --git a/assets/js/worker.js b/assets/js/worker.js index 7af140584..e463940af 100644 --- a/assets/js/worker.js +++ b/assets/js/worker.js @@ -1,15 +1,24 @@ +/** + * This worker is responsible for loading the lunr index, searching the index and returning the results. + * + * It has 2 action types, 'init' and 'search'. + * + * 'init' action is used to load the lunr index and the raw index data. + * 'search' action is used to search the lunr index with the provided query. + */ + if (typeof importScripts === 'function') { importScripts('https://unpkg.com/lunr@2.3.8/lunr.min.js'); } let idx; const versionRegex = new RegExp("^\/docs\/([0-9\.]*|latest)\/"); -const resultDetails = new Map(); // Will hold the data for the search results (titles and summaries) +const resultDetails = new Map(); let indexReadyPromise; -// Initialize the index self.onmessage = async function (event) { if (event.data.type === 'init') { + // Initialize the lunr index and the raw index data. indexReadyPromise = new Promise(async (resolve, reject) => { try { const rawIndex = await fetch(event.data.rawIndexUrl); @@ -33,6 +42,7 @@ self.onmessage = async function (event) { } }); } else if (event.data.type === 'search') { + // Search the lunr index with the provided query. try { await indexReadyPromise; const regexResults = versionRegex.exec(event.data.currentPath);