perf(HTML static search): reduce the page load size by using IndexedDB storage

strictdoc/core/project_config.py

@@ -412,6 +412,10 @@ def get_static_files_path(self) -> str:
     def get_extra_static_files_path(self) -> str:
         return self.environment.get_extra_static_files_path()
 
+    def get_project_hash(self) -> str:
+        assert self.input_paths is not None and len(self.input_paths) > 0
+        return get_md5(self.input_paths[0])
+
     def shall_parse_nodes(self, path_to_file: str) -> bool:
         if self.source_root_path is None:
             return False

strictdoc/core/traceability_index.py

@@ -68,6 +68,15 @@ def __init__(
             datetime.datetime.fromtimestamp(0)
         )
 
+        # The timestamp is used by HTML/JS for invalidating the search index
+        # cache in the IndexedDB database.
+        # If no documents have to be re-generated with the second+ run of
+        # StrictDoc, this timestamp is set of a modification date of the first
+        # SDoc document, see export_static_html_search_index.
+        self.search_index_timestamp: datetime.datetime = datetime.datetime.now(
+            datetime.timezone.utc
+        )
+
     @property
     def document_iterators(self) -> Dict[SDocDocument, DocumentCachingIterator]:
         return self._document_iterators

strictdoc/export/html/_static/static_html_search.js

@@ -223,9 +223,13 @@ userinput.addEventListener("focus", handleInputEvent_focus, true);
 const searchResultsView = new SearchResultsView();
 
 function handleInputEvent_input() {
+    if (!window.SDOC_SEARCH_INDEX || !window.SDOC_MAP_MID_TO_NODES) {
+        console.log("Search: Cannot perform search: Search index is not available yet.")
+        return;
+    }
+
     if (userinput.value === "") {
         userinput.dataset.prevValue = "";
-        // searchResultsView.clearResults
         searchResultsView.hideResults();
         return;
     }
@@ -283,7 +287,7 @@ function handleInputEvent_input() {
         if (!searchQuery.includes('"')) {
             let uniqueResults = new Set();
             for (const token of queryDict["terms"]) {
-                const tokenResults = window.SDOC_LUNR_SEARCH_INDEX[token];
+                const tokenResults = window.SDOC_SEARCH_INDEX[token];
                 if (tokenResults) {
                     uniqueResults = new Set([...uniqueResults, ...tokenResults]);
                 }
@@ -292,13 +296,13 @@ function handleInputEvent_input() {
         }
     } else {
         const firstTerm = queryDict["terms"][0];
-        const firstTermResults = window.SDOC_LUNR_SEARCH_INDEX[firstTerm];
+        const firstTermResults = window.SDOC_SEARCH_INDEX[firstTerm];
         if (firstTermResults && firstTermResults.length > 0) {
             let uniqueResults = new Set(firstTermResults);
 
             if (queryDict["terms"].length > 1) {
                 for (let i = 1; i < queryDict["terms"].length; i++) {
-                    const termResults = window.SDOC_LUNR_SEARCH_INDEX[queryDict["terms"][i]];
+                    const termResults = window.SDOC_SEARCH_INDEX[queryDict["terms"][i]];
                     const termUniqueResults = new Set(termResults);
 
                     uniqueResults = intersectSets([uniqueResults, termUniqueResults]);
@@ -371,3 +375,120 @@ function handleInputEvent_keyUp(event) {
         }
     }
 }
+
+window.addEventListener("load", async () => {
+  const DB_VERSION = 1;
+  const timestampMeta = document.querySelector(
+    'meta[name="strictdoc-search-index-timestamp"]'
+  )?.content;
+  const projectHash = document.querySelector(
+    'meta[name="strictdoc-project-hash"]'
+  )?.content;
+  const pathToSearchIndex = document.querySelector(
+    'meta[name="strictdoc-search-index-path"]'
+  )?.content;
+
+  if (!projectHash || !pathToSearchIndex || !timestampMeta) {
+    console.error("Search: Missing required meta tags!");
+    return;
+  }
+
+  const dbName = "strictdoc_search_index_" + projectHash;
+
+  const openDB = (name, version = 1) =>
+    new Promise((resolve, reject) => {
+      const request = indexedDB.open(name, version);
+      request.onupgradeneeded = (e) => {
+        const db = e.target.result;
+        db.createObjectStore("indexes", { keyPath: "name" });
+      };
+      request.onsuccess = () => resolve(request.result);
+      request.onerror = () => reject(request.error);
+    });
+
+  const deleteDB = (name) =>
+    new Promise((resolve, reject) => {
+      const delReq = indexedDB.deleteDatabase(name);
+      delReq.onsuccess = () => resolve();
+      delReq.onerror = () => reject(delReq.error);
+    });
+
+  const getFromStore = (db, storeName, key) =>
+    new Promise((resolve, reject) => {
+      const tx = db.transaction(storeName, "readonly");
+      const store = tx.objectStore(storeName);
+      const req = store.get(key);
+      req.onsuccess = () => resolve(req.result);
+      req.onerror = () => reject(req.error);
+    });
+
+  const saveToStore = (db, storeName, items) =>
+    new Promise((resolve, reject) => {
+      const tx = db.transaction(storeName, "readwrite");
+      const store = tx.objectStore(storeName);
+      items.forEach((item) => store.put(item));
+      tx.oncomplete = () => resolve();
+      tx.onerror = () => reject(tx.error);
+    });
+
+  const loadScript = (url) =>
+    new Promise((resolve, reject) => {
+      const script = document.createElement("script");
+      script.src = url;
+      script.onload = () => resolve();
+      script.onerror = () => reject(new Error(`Failed to load script ${url}`));
+      document.head.appendChild(script);
+    });
+
+  try {
+    console.log("Search: LOAD_DB_INDEX: Start");
+    const db = await openDB(dbName, DB_VERSION);
+    const tsEntry = await getFromStore(db, "indexes", "TIMESTAMP");
+
+    if (tsEntry && tsEntry.value === timestampMeta) {
+
+      console.time("Search: LOAD_DB_INDEX");
+      const lunrEntry = await getFromStore(db, "indexes", "SDOC_SEARCH_INDEX");
+      const nodesEntry = await getFromStore(db, "indexes", "SDOC_MAP_MID_TO_NODES");
+      console.timeEnd("Search: LOAD_DB_INDEX");
+
+      if (lunrEntry && nodesEntry) {
+        window.SDOC_SEARCH_INDEX = lunrEntry.value;
+        window.SDOC_MAP_MID_TO_NODES = nodesEntry.value;
+        return;
+      }
+    }
+
+    db.close();
+    await deleteDB(dbName);
+
+    try {
+      console.time("Search: LOAD_JS_INDEX");
+      await loadScript(new URL(pathToSearchIndex, window.location.href).href);
+      console.timeEnd("Search: LOAD_JS_INDEX");
+      console.log("Search: JS search index loaded successfully.");
+    } catch (e) {
+      console.error("Search: Failed to load JS search index script:", e);
+      return;
+    }
+
+    console.time("Search: SAVE_DB_INDEX");
+    const newDB = await openDB(dbName, DB_VERSION);
+    await saveToStore(newDB, "indexes", [
+      { name: "SDOC_SEARCH_INDEX", value: window.SDOC_SEARCH_INDEX },
+      { name: "SDOC_MAP_MID_TO_NODES", value: window.SDOC_MAP_MID_TO_NODES },
+      { name: "TIMESTAMP", value: timestampMeta },
+    ]);
+    console.timeEnd("Search: SAVE_DB_INDEX");
+
+  } catch (err) {
+    console.error("Search: Error loading search index:", err);
+
+    try {
+      await loadScript(new URL(pathToSearchIndex, window.location.href).href);
+      console.log("Search: Script loaded without IndexedDB fallback");
+    } catch (e) {
+      console.error("Search: Failed to load search index script:", e);
+    }
+  }
+});

strictdoc/export/html/html_generator.py

@@ -692,6 +692,24 @@ def export_static_html_search_index(
                 "All documents are up-to-date. "
                 "Skipping the generation of a search index."
             )
+            # If no documents need to be regenerated, set the search_index_timestamp
+            # to the timestamp of the first document. The assumption here is
+            # that StrictDoc does not randomize the document list, and the first
+            # document will always be the same.
+            # The HTML/JS code can rely on this timestamp to decide whether it
+            # has to re-read the search index from the JS file or it can simply
+            # fetch it from the DB which is 2x faster when it comes to very
+            # large indexes.
+            if len(traceability_index.document_tree.document_list) > 0:
+                first_document = traceability_index.document_tree.document_list[
+                    0
+                ]
+                assert first_document.meta is not None
+                traceability_index.search_index_timestamp = (
+                    get_file_modification_time(
+                        first_document.meta.input_doc_full_path
+                    )
+                )
             return
 
         global_index: Dict[str, Set[int]] = defaultdict(set)
@@ -747,7 +765,7 @@ def default(obj: Any) -> Any:
 
         with measure_performance("Serialize search index to JS"):
             document_content = (
-                b"window.SDOC_LUNR_SEARCH_INDEX = "
+                b"window.SDOC_SEARCH_INDEX = "
                 + orjson.dumps(
                     global_index,
                     option=orjson.OPT_NON_STR_KEYS,

strictdoc/export/html/templates/_shared/static_search_head.jinja

@@ -0,0 +1,5 @@
+<meta name="strictdoc-project-hash" content="{{ view_object.project_config.get_project_hash() }}">
+<meta name="strictdoc-search-index-timestamp" content="{{ view_object.traceability_index.search_index_timestamp.timestamp() }}">
+<meta name="strictdoc-search-index-path" content="{{ view_object.render_static_url('static_html_search_index.js') }}">
+
+<script src="{{ view_object.render_static_url('static_html_search.js') }}" defer></script>

strictdoc/export/html/templates/base.jinja.html

@@ -106,10 +106,5 @@
   <div id="confirm"></div>
   {% block scripts %}{% endblock scripts %}
 
-  {% if not view_object.project_config.is_running_on_server %}
-    <script src="{{ view_object.render_static_url('static_html_search_index.js') }}" defer></script>
-    <script src="{{ view_object.render_static_url('static_html_search.js') }}" defer></script>
-  {% endif %}
-
 </body>
 </html>

strictdoc/export/html/templates/screens/document/document/index.jinja

@@ -22,6 +22,10 @@
   <script src="{{ view_object.render_static_url('controllers/copy_stable_link_button_controller.js') }}"></script>
   <script src="{{ view_object.render_static_url('controllers/copy_to_clipboard_controller.js') }}"></script>
 
+  {% if not view_object.is_running_on_server and not view_object.standalone %}
+    {% include "_shared/static_search_head.jinja" %}
+  {% endif %}
+
   {%- if view_object.is_running_on_server and not view_object.standalone -%}
   <script type="module">
     import hotwiredTurbo from "{{ view_object.render_static_url('turbo.min.js') }}";

strictdoc/export/html/templates/screens/project_index/index.jinja

@@ -24,6 +24,11 @@
        fine-grained approach.
   #}
   <script src="{{ view_object.render_static_url('project_tree.js') }}"></script>
+
+  {% if not view_object.is_running_on_server %}
+    {% include "_shared/static_search_head.jinja" %}
+  {% endif %}
+
 {%- if view_object.project_config.is_running_on_server and not view_object.standalone -%}
   <script type="module">
     import hotwiredTurbo from "{{ view_object.render_static_url_with_prefix('turbo.min.js') }}";