From 180d95ce10d881299a60ac89af600e23f94b6faa Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 13:25:37 +0800
Subject: [PATCH 01/11] added rewrited trie implementation, manipulation
 functions, and search data, maintianing funcitons and writing funciton

---
 javascript/searchRewrite.js | 307 ++++++++++++++++++++++++++++++++++++
 1 file changed, 307 insertions(+)
 create mode 100644 javascript/searchRewrite.js

diff --git a/javascript/searchRewrite.js b/javascript/searchRewrite.js
new file mode 100644
index 000000000..e3db88768
--- /dev/null
+++ b/javascript/searchRewrite.js
@@ -0,0 +1,307 @@
+import fs from "fs";
+
+// line 3 to 68: trie implementation and search functions
+class trieNode {
+    constructor() {
+        this.children = {};
+        this.value = [];
+        this.key = "";
+    }
+}
+
+export function insert(keyStr, value, trie) {
+    const keys = [...keyStr];
+    let node = trie;
+    for (let i = 0; i < keys.length; i++) {
+        if (!node.children[keys[i]]) {
+            node.children[keys[i]] = new trieNode();
+        }
+        node = node.children[keys[i]];
+    }
+    node.value.push(value);
+    node.key = keyStr;
+}
+
+export function search(keyStr, trie) {
+    const keys = [...keyStr];
+    let node = trie;
+    for (let i = 0; i < keys.length; i++) {
+        if(node === undefined || node.children === undefined) {
+            console.log("when searching, got undefined node or node.children");
+            console.log("i is " + i);
+            return null;
+        }
+
+        if (!node.children[keys[i]]) {
+            return null;
+        }
+        node = node.children[keys[i]];
+    }
+    return node.value;
+}
+
+export function autoComplete(incompleteKeys, trie, n = 30) {
+    let node = trie;
+    for (let i = 0; i < incompleteKeys.length; i++) {
+        if (!node.children[incompleteKeys[i]]) {
+            return [];
+        }
+        node = node.children[incompleteKeys[i]];
+    }
+    const result = [];
+    const queue = [node];
+    while (queue.length > 0 && result.length < n) {
+        const node = queue.shift();
+        if (node.value.length > 0) {
+            result.push(node.key);
+        }
+        for (const child of Object.values(node.children)) {
+            queue.push(child);
+        }
+    }
+    return result;
+}
+
+export const getUrl= searchResult => `https://sourceacademy.nus.edu.sg/sicpjs/${searchResult.id}`;
+
+
+// search data, maintaining and updation functions and write function from this line onwards
+export const idToContentMap = {};
+export const textTrie = new trieNode();
+export const indexTrie = new trieNode();
+
+
+const parseIndexSearchEntryTo = (node, json) => {
+    if (node === null) return;
+    if(indexParsers[node.nodeName]) {
+        indexParsers[node.nodeName](node, json);
+        return;
+    }
+};
+
+// currently not used, a map of all the tags in json, good reference
+const processingFunctions = {
+    '#text': 1,
+  
+    B: 1,
+    BR: 1,
+  
+    DISPLAYFOOTNOTE: 1,
+  
+    EM: 1,
+  
+    EPIGRAPH: 1,
+  
+    EXERCISE: 1,
+  
+    FIGURE: 1,
+  
+    FOOTNOTE_REF: 1,
+  
+    JAVASCRIPTINLINE: 1,
+  
+    LATEX: 1,
+  
+    LI: 1,
+  
+    LINK: 1,
+  
+    META: 1,
+  
+    OL: 1,
+  
+    REF: 1,
+  
+    REFERENCE: 1,
+  
+    SNIPPET: 1,
+  
+    SUBHEADING: 1,
+  
+    SUBSUBHEADING: 1,
+  
+    TABLE: 1,
+  
+    TEXT: 1,
+  
+    TITLE:1,
+  
+    TT: 1,
+  
+    UL: 1,
+};
+
+const indexParsers = {
+    // plain text nodes
+     "#text": (node,json) => {
+        json["text"] += node.nodeValue;
+    },
+    
+    OPERATOR: (node,json) => {
+        json["text"] += "operators";
+    },
+    PARSING: (node,json) => {
+        json["text"] += "parsing JavaScript";
+    },
+    FUNCTION: (node,json) => {
+        json["text"] += "function (JavaScript)";
+    }, 
+    PRIMITIVE: (node,json) => {
+        json["text"] += "primitive functions (ECMAScript equivalent in parentheses if they are in the ECMAScript standard)";
+    },
+    ENDASH: (node,json) => {
+        json["text"] += "–";
+    },
+    APOS: (node,json) => {
+        json["text"] += "'";
+    },
+    EACUTE_LOWER : (node,json) => {
+        json["text"] += "é";
+    },
+    AACUTE_LOWER : (node,json) => {
+        json["text"] += "á";
+    },
+    AACUTE_UPPER : (node,json) => {
+        json["text"] += "Á";
+    },
+    SPACE: (node,json) => {
+        json["text"] += " ";
+    },
+    // next and only child is text nodes
+    ECMA: (node,json) => {
+        if(node.firstChild.nodeName !== "#text") {
+            console.log("when parsing ECMA, got this unknown node name" + node.firstChild.nodeName);
+            return;
+        }
+        json["text"] += ` (${node.firstChild.nodeValue})`;
+    },
+    JAVASCRIPTINLINE: (node,json) => {
+        if(node.firstChild.nodeName !== "#text") {
+            console.log("when parsing JAVASCRIPTINLINE, got this unknown node name" + node.firstChild.nodeName);
+            return;
+        }
+        json["text"] += node.firstChild.nodeValue;
+    },
+    QUOTE: (node,json) => {
+        if(node.firstChild.nodeName !== "#text") {
+            console.log("when parsing QUOTE, got this unknown node name" + node.firstChild.nodeName);
+            return;
+        }
+        json["text"] += `"${node.firstChild.nodeValue}"`;
+    },
+    USE: (node,json) => {
+        if(node.firstChild.nodeName !== "#text") {
+            console.log("when parsing USE, got this unknown node name" + node.firstChild.nodeName);
+            return;
+        }
+        json["text"] += node.firstChild.nodeValue;
+        json["ORDER"] = node.firstChild.nodeValue;
+    },
+    DECLARATION: (node,json) => {
+        if(node.firstChild.nodeName !== "#text") {
+            console.log("when parsing USE, got this unknown node name" + node.firstChild.nodeName);
+            return;
+        }
+        json["text"] += node.firstChild.nodeValue;
+        json["ORDER"] = node.firstChild.nodeValue;
+    },
+    ORDER: (node,json) => {
+        if(node.firstChild.nodeName !== "#text") {
+            console.log("when parsing ORDER, got this unknown node name" + node.firstChild.nodeName);
+            return;
+        }
+        json["ORDER"] = node.firstChild.nodeValue;
+    },
+    // other nodes
+    CLOSE: (node,json) => {
+        json["CLOSE"] = true;
+    },
+    OPEN: (node,json) => {
+        json["OPEN"] = true;
+    },
+    SUBINDEX: (node, json) => {
+        const newJson = {"text":""};
+        json["SUBINDEX"] = newJson;
+        for (let i = 0; i < node.childNodes.length; i++) {
+            const child = node.childNodes[i];
+            parseIndexSearchEntryTo(child, newJson);
+        }
+    },
+    SPLITINLINE: (node,json) => {
+        const javascriptNode = node.getElementsByTagName("JAVASCRIPT")[0];
+        if(!javascriptNode) {
+            console.log("when parsing SPLITINLINE, got no JAVASCRIPT node");
+            return;
+        }
+        for (let i = 0; i < node.childNodes.length; i++) {
+            const child = node.childNodes[i];
+            parseIndexSearchEntryTo(child, json);
+        }
+    },
+    INDEX: (node, json) => {
+        json["text"] = "";
+        for (let i = 0; i < node.childNodes.length; i++) {
+            const child = node.childNodes[i];
+            parseIndexSearchEntryTo(child, json);
+        }
+        if(node.getElementsByTagName("PRIMITIVE")[0]) {
+         json.SUBINDEX.text = json.SUBINDEX.text.replace(" (\\textit{ns})", "");
+        }
+    },
+
+    //todo
+    LATEXINLINE: (node,json) => {
+        json["text"] += `LATEX: ${node.firstChild.nodeValue}`;
+    },  
+};
+
+export const parseAndInsertToIndexTrie = (node, json) => {
+    parseIndexSearchEntryTo(node, json);
+    insert(json.text, json, indexTrie);
+}
+
+export const parseAndInsertToIdToContentMap = (json,chapterIndex, idStack = []) => {
+    if(Array.isArray(json)) {
+        for (let i = 0; i < json.length; i++) {
+        parseAndInsertToIdToContentMap(json[i],chapterIndex,idStack);
+        }
+        return;
+    }
+
+    if(json.id) {
+        const id = chapterIndex + json.id;
+        idStack.push(id);
+        idToContentMap[id] = "";
+    }
+    if(json.body) {
+        idToContentMap[idStack[idStack.length-1]] += json.body;
+    }
+    if(json.child) {
+        parseAndInsertToIdToContentMap(json.child,chapterIndex,idStack);
+    }
+    if(json.id) {
+        idStack.pop();
+    }
+}
+
+const buildTextTrie = () => {
+    for (const [key, value] of Object.entries(idToContentMap)) {
+        const words = value.trim().replace(/\s+/g," ").toLowerCase().replace(/\n/gi, "").split(" ").filter(a => a != "" && a != '\"');
+        for (const word of words) {
+            insert(word, key, textTrie);
+        }
+    }
+}
+
+export const writeRewritedSearchData = () => {
+    buildTextTrie();
+
+    const searchData = {indexTrie, textTrie, idToContentMap};
+    fs.writeFile("rewritedSearchData.json", JSON.stringify(searchData), (err) => {
+        if (err) {
+            console.log(err);
+        }
+    });
+}
+
+

From 33115fe238258f9812a7996ef1d4f20a305a59c3 Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 13:52:04 +0800
Subject: [PATCH 02/11] integrate rewriteSeatch to index and parseXmlJson

---
 javascript/index.js         |  4 +++-
 javascript/parseXmlJson.js  | 31 +++++++++++++++++++++++++++++++
 javascript/searchRewrite.js |  4 +++-
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/javascript/index.js b/javascript/index.js
index 89fef6feb..0c8e9fc42 100644
--- a/javascript/index.js
+++ b/javascript/index.js
@@ -37,6 +37,7 @@ import { getAnswers } from "./processingFunctions/processExercisePdf";
 
 // json (for cadet frontend)
 import { parseXmlJson } from "./parseXmlJson";
+import {writeRewritedSearchData} from "./searchRewrite";
 import { setupSnippetsJson } from "./processingFunctions/processSnippetJson";
 import { createTocJson } from "./generateTocJson";
 import { setupReferencesJson } from "./processingFunctions/processReferenceJson";
@@ -360,7 +361,8 @@ async function main() {
     await recursiveXmlToHtmlInOrder("setupSnippet");
     console.log("setup snippets and references done\n");
 
-    recursiveXmlToHtmlInOrder("parseXml");
+    await recursiveXmlToHtmlInOrder("parseXml");
+    writeRewritedSearchData();
   }
 }
 
diff --git a/javascript/parseXmlJson.js b/javascript/parseXmlJson.js
index e27e1a516..35a9cad49 100644
--- a/javascript/parseXmlJson.js
+++ b/javascript/parseXmlJson.js
@@ -16,6 +16,8 @@ import {
   generateSearchData
 } from "./generateSearchData";
 
+import {parseAndInsertToIndexTrie, parseAndInsertToIdToContentMap} from "./searchRewrite";
+
 let paragraph_count = 0;
 let heading_count = 0;
 let footnote_count = 0;
@@ -121,6 +123,30 @@ const processLatex = (node, obj, inline) => {
   obj["tag"] = "LATEX";
 };
 
+const tagsWithIds = {
+  TITLE: () => subsubsection_count>0? `#sec${chapterIndex}.${subsubsection_count}` :"",
+  TEXT:() => "#p" + paragraph_count,
+  SUBHEADING: () => `#h${heading_count}`,
+  SUBSUBHEADING: () => `#h${heading_count}`,
+  SECTION: () => `#h${heading_count}`,
+  FOOTNOTE: () => `#footnote-link-${footnote_count}`,
+  DISPLAYFOOTNOTE: () => `#footnote-${display_footnote_count}`,
+  SNIPPET: () => `{snippet_count}`,
+  //todo, fix this
+  EXERCISE: () => `#ex-1.${exercise_count}`,
+  DISPLAYFOOTNOTE: () => `#footnote-${display_footnote_count}`,
+};
+const findParentID = (node) => {
+  let parent = node.parentNode;
+  while (parent) {
+    if(tagsWithIds[parent.nodeName]) {
+      return `${chapterIndex}` + tagsWithIds[parent.nodeName]();
+    } else {
+      parent = parent.parentNode;
+    }
+  }
+}
+
 const processTextFunctions = {
   // Text tags: tag that is parsed as text
   "#text": (node, obj) => {
@@ -132,6 +158,10 @@ const processTextFunctions = {
       }
     }
   },
+  INDEX: (node, obj) => {
+    const id = findParentID(node);
+    parseAndInsertToIndexTrie(node, {id});
+  },
 
   AMP: (_node, obj) => {
     processText("&amp;", obj);
@@ -571,6 +601,7 @@ export const parseXmlJson = (doc, arr, filename) => {
     recursiveProcessTextJson(name.nextSibling, arr, title);
   }
 
+  parseAndInsertToIdToContentMap(arr,chapterIndex);
   generateSearchData(doc, filename);
 
 };
diff --git a/javascript/searchRewrite.js b/javascript/searchRewrite.js
index e3db88768..9270faf9c 100644
--- a/javascript/searchRewrite.js
+++ b/javascript/searchRewrite.js
@@ -130,6 +130,8 @@ const processingFunctions = {
   
     UL: 1,
 };
+// this list is dependent on the implementation of parseXmlJson and frontend
+// so why parsing json at frontend? why not send html to frontend which will then only need to care
 
 const indexParsers = {
     // plain text nodes
@@ -297,7 +299,7 @@ export const writeRewritedSearchData = () => {
     buildTextTrie();
 
     const searchData = {indexTrie, textTrie, idToContentMap};
-    fs.writeFile("rewritedSearchData.json", JSON.stringify(searchData), (err) => {
+    fs.writeFile("json/rewritedSearchData.json", JSON.stringify(searchData), (err) => {
         if (err) {
             console.log(err);
         }

From d31b21b2dc6e4ab1fba5ebc7d4df1e37cb5dd138 Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 14:55:11 +0800
Subject: [PATCH 03/11] added testcase for the index search

---
 failedTests.txt                 |  32 +++++
 javascript/index.js             |   2 +
 javascript/searchRewriteTest.js | 241 ++++++++++++++++++++++++++++++++
 3 files changed, 275 insertions(+)
 create mode 100644 failedTests.txt
 create mode 100644 javascript/searchRewriteTest.js

diff --git a/failedTests.txt b/failedTests.txt
new file mode 100644
index 000000000..e866c2a57
--- /dev/null
+++ b/failedTests.txt
@@ -0,0 +1,32 @@
+abs: result is 0, expected occuer number is: 1
+accelerated_sequence: result is null, expected occuer number is: 1
+accumulate: result is 4, expected occuer number is: 3
+accumulate_n: result is null, expected occuer number is: 1
+accumulator: result is 3, expected occuer number is: 2
+Ackermann’s function: result is null, expected occuer number is: 1
+actual_value: result is null, expected occuer number is: 1
+Ada: result is 2, expected occuer number is: 1
+add (generic): result is 2, expected occuer number is: 1
+add_action: result is 1, expected occuer number is: 2
+add_complex: result is null, expected occuer number is: 1
+add_complex_to_javascript_num: result is null, expected occuer number is: 1
+addend: result is null, expected occuer number is: 1
+adder (primitive constraint): result is null, expected occuer number is: 1
+adder: result is 4, expected occuer number is: 3
+add_interval: result is null, expected occuer number is: 1
+add_lists: result is null, expected occuer number is: 1
+add_poly: result is null, expected occuer number is: 1
+add_rat: result is null, expected occuer number is: 1
+add_rule_or_assertion: result is null, expected occuer number is: 1
+add_streams: result is null, expected occuer number is: 1
+add_terms: result is null, expected occuer number is: 1
+add_to_agenda: result is 1, expected occuer number is: 2
+add_vect: result is 2, expected occuer number is: 1
+adjoin_arg: result is null, expected occuer number is: 1
+adjoin_set: result is 3, expected occuer number is: 5
+adjoin_term: result is 1, expected occuer number is: 2
+advance_pc: result is null, expected occuer number is: 1
+after_delay: result is 1, expected occuer number is: 2
+A’h-mose: result is null, expected occuer number is: 1
+algebra expression: result is null, expected occuer number is: 7
+algebraci specification for data: result is null, expected occuer number is: 1
\ No newline at end of file
diff --git a/javascript/index.js b/javascript/index.js
index 0c8e9fc42..56af1c035 100644
--- a/javascript/index.js
+++ b/javascript/index.js
@@ -36,6 +36,7 @@ import { setupSnippetsJs } from "./processingFunctions/processSnippetJs";
 import { getAnswers } from "./processingFunctions/processExercisePdf";
 
 // json (for cadet frontend)
+import {testIndexSearch} from "./searchRewriteTest";
 import { parseXmlJson } from "./parseXmlJson";
 import {writeRewritedSearchData} from "./searchRewrite";
 import { setupSnippetsJson } from "./processingFunctions/processSnippetJson";
@@ -363,6 +364,7 @@ async function main() {
 
     await recursiveXmlToHtmlInOrder("parseXml");
     writeRewritedSearchData();
+    testIndexSearch();
   }
 }
 
diff --git a/javascript/searchRewriteTest.js b/javascript/searchRewriteTest.js
new file mode 100644
index 000000000..789ad4ebd
--- /dev/null
+++ b/javascript/searchRewriteTest.js
@@ -0,0 +1,241 @@
+// do not know why, I can not run yarn test on my device which leads to binding error.
+// this file is meant to be temporary
+
+
+/* known issues: 
+did not process the "seexml" file, so no see also in index
+did not process the latex, roman, italic, etc.
+*/
+
+/*
+abs, 14
+absolute value, 13
+abstract data, 72, see also data abstraction abstraction, see also data abstraction;
+higher-order functions; means of
+abstraction
+common pattern and, 50
+functional, 22
+metalinguistic, 318
+in register-machine design, 456–457 of search in nondeterministic
+programming, 378
+abstraction barriers, 71, 76–78, 147
+in complex-number system, 148
+in generic arithmetic system, 164
+in query language, 437
+in representing JavaScript syntax, 329
+abstract models for data, 78 n abstract syntax
+in metacircular evaluator, 322
+in query interpreter, 425 accelerated_sequence, 297 accumulate, 53 (ex. 1.32), 100
+same as fold_right, 105 (ex. 2.38) accumulate_n, 104 (ex. 2.36) accumulator, 100, 196 (ex. 3.1) Ackermann’s function, 31 (ex. 1.10) acquire a mutex, 276
+actions, in register machine, 454–455
+actual_value, 364
+Ada, 411 (ex. 4.61)
+Adams, Norman I., IV, 356 n add (generic), 165
+used for polynomial coefficients, 179, 180
+add_action, 244, 247
+add_complex, 150 add_complex_to_javascript_num, 169 addend, 128
+adder (primitive constraint), 256 adder
+full, 243
+half, 242
+ripple-carry, 245 (ex. 3.30)
+add_interval, 81
+additivity, 72, 147, 156–162, 166 add_lists, 371
+add_poly, 178
+add_rat, 73
+address, 488
+address arithmetic, 488 add_rule_or_assertion, 434 add_streams, 290
+add_terms, 179 add_to_agenda, 248, 251 add_vect, 118 (ex. 2.46) adjoin_arg, 505 n
+adjoining to a list with pair, 88 adjoin_set, 131
+binary-tree representation, 136 ordered-list representation, 135 (ex.
+2.61)
+unordered-list representation, 132 for weighted sets, 145
+adjoin_term, 179, 182
+Adleman, Leonard, 46 n
+administrative assistant, importance of,
+403 advance_pc, 479
+after_delay, 244, 248
+agenda, see digital-circuit simulation A’h-mose, 40 n
+algebra, symbolic, see symbolic algebra algebraic expression, 176
+differentiating, 126–131 representing, 128–131 simplifying, 129–130
+algebraic specification for data, 79 n
+Algol
+block structure, 26
+call-by-name argument passing, 286 n,
+363 n
+thunks, 286 n, 363 n
+algorithm
+optimal, 104 n probabilistic, 45–46, 188 n
+aliasing, 204 n
+Al-Karaji, 36 n
+Allen, John, 494 n all_regs (compiler), 542 n alternative
+of conditional expression, 14
+of conditional statement, 57 always_true, 428
+amb, 374
+amb evaluator, see nondeterministic
+evaluator ambeval, 388
+analog computer, 306 (fig. 3.34)
+analyze
+metacircular, 356
+nondeterministic, 387
+analyze_...
+metacircular, 356–359, 360 (ex. 4.21)
+nondeterministic, 389–392 analyze_amb, 394
+analyzing evaluator, 355–360
+as basis for nondeterministic evaluator, 386
+and (query language), 405
+evaluation of, 413, 426, 446 (ex. 4.73)
+and-gate, 241 and_gate, 245
+an_element_of, 375 angle
+data-directed, 160
+polar representation, 152 rectangular representation, 151 with tagged data, 154
+angle_polar, 154 angle_rectangular, 153 an_integer_starting_from, 375 Appel, Andrew W., 541 n
+append, 88, 225 (ex. 3.12)
+as accumulation, 103 (ex. 2.33) append_mutator vs., 225 (ex. 3.12) as register machine, 492 (ex. 5.21) “what is” (rules) vs. “how to”
+(function), 399–400
+append_instruction_sequences, 524, 544
+append_mutator, 225 (ex. 3.12)
+as register machine, 492 (ex. 5.21)
+append_to_form (rules), 410 applicative-order evaluation, 13
+in JavaScript, 13
+normal order vs., 17 (ex. 1.5), 43 (ex.
+1.20), 361–362 apply (lazy), 364
+apply (metacircular), 324 tail recursion and, 324 n
+apply (primitive method), 346 n apply_a_rule, 430 apply_dispatch, 507
+modified for compiled code, 558 apply_generic, 160
+with coercion, 171, 174 (ex. 2.81)
+with coercion by raising, 175 (ex. 2.84) with coercion of multiple arguments,
+175 (ex. 2.82)
+with coercion to simplify, 176 (ex.
+2.85)
+with message passing, 163 with tower of types, 173
+apply_in_underlying_javascript, 159 n, 346 n
+apply_primitive_function, 324, 340, 346
+apply_rules, 430 arbiter, 278 n arctangent, 151 n arg_expressions, 331 argl register, 500 argument(s), 9
+arbitrary number of, 276
+delayed, 306
+argument passing, see call-by-name
+argument passing; call-by-need
+argument passing Aristotle’s De caelo (Buridan’s
+commentary on), 278 n arithmetic
+address arithmetic, 488 generic, 163, see also generic
+arithmetic operations
+on complex numbers, 148
+on intervals, 81–84
+on polynomials, see polynomial
+arithmetic
+on power series, 294 (ex. 3.60), 295
+(ex. 3.62)
+on rational numbers, 72–76
+operators for, 4
+array, see vector (data structure) arrow function, see lambda expression articles, 381
+ASCII code, 140
+assemble, 474, 475 n
+assembler, 470, 474–477
+assert (query interpreter), 419 assertion, 401
+implicit, 407 assertion_body, 443
+assign (in register machine), 453
+instruction constructor, 478 simulating, 478
+storing label in register, 459
+assignment, 190–206
+assignment expression, 192 assignment operation, 190
+benefits of, 197–200
+bugs associated with, 204 n, 205 constant/variable declaration vs., 192 n costs of, 200–206
+equality test vs., 192 n
+evaluation of, 210
+parsing of, 333
+value of, 192 n
+assignment_symbol, 333 assignment_value_expression, 333 assign_reg_name, 478 assign_symbol_value, 341, 342 assign_value_exp, 478
+assoc, 236 associativity
+of conditional expression, 14
+of operators, 5
+atomic operations supported in hardware,
+278 n
+atomic requirement for test_and_set, 277 attach_tag, 152
+using JavaScript data types, 168 (ex. 2.78)
+augend, 128
+automagically, 376
+automatic search, 373, see also search
+history of, 376 n
+automatic storage allocation, 487 average, 19
+average_damp, 63
+average damping, 61
+averager (constraint), 261 (ex. 3.33)
+*/
+
+// I include 2 out of the 5 columns of all the index with A here.
+
+import { indexTrie, search } from "./searchRewrite"
+import fs from "fs";
+
+const indexSearchTestCase = {
+   "abs": 1,
+   "absolute value": 1,
+    "abstract data": 1,
+    "abstraction": 6,
+    "abstraction barriers": 8,
+    "abstract models for data": 1,
+    "abstract syntax": 2,
+    "accelerated_sequence": 1,
+    "accumulate": 3,
+    "accumulate_n": 1,
+    "accumulator": 2,
+    "Ackermann’s function": 1,
+    "acquire a mutex": 1,
+    "actions, in register machine": 2,
+    "actual_value": 1,
+    "Ada": 1,
+    "Adams, Norman I., IV": 1,
+    "add (generic)": 1,
+    "add_action": 2,
+    "add_complex": 1,
+    "add_complex_to_javascript_num": 1,
+    "addend": 1,
+    "adder (primitive constraint)": 1,
+    "adder": 3,
+    "add_interval": 1,
+    "additivity": 5,
+    "add_lists": 1,
+    "add_poly": 1,
+    "add_rat": 1,
+    "address": 1,
+    "address arithmetic": 1,
+    "add_rule_or_assertion": 1,
+    "add_streams": 1,
+    "add_terms": 1,
+    "add_to_agenda": 2,
+    "add_vect": 1,
+    "adjoin_arg": 1,
+    "adjoining to a list with pair": 1,
+    "adjoin_set": 5,
+    "adjoin_term": 2,
+    "Adleman, Leonard": 1,
+    "administrative assistant, importance of": 1,
+    "advance_pc": 1,
+    "after_delay": 2,
+    "A’h-mose": 1,
+    "algebra expression": 7,
+    "algebraci specification for data": 1,
+}
+
+const failedTests = [];
+const writeFailureMessage = (key, searchResult) => {
+    failedTests.push(`${key}: result is ${searchResult}, expected occuer number is: ${indexSearchTestCase[key]}`);
+}
+
+
+export function testIndexSearch() {
+    for (const [key, value] of Object.entries(indexSearchTestCase)) {
+        const result = search(key, indexTrie);
+        console.log(result);
+        if (result === null) {
+            writeFailureMessage(key, "null");
+            continue;
+        }
+        if (result.length !== value) {
+            writeFailureMessage(key, result.length);
+            continue;
+        }
+    }
+    fs.writeFileSync("failedTests.txt", failedTests.join("\n"));
+}
+

From 9bd95a759c92e8210de9fa23ed975273aea16baf Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 15:27:09 +0800
Subject: [PATCH 04/11] debugged and fix the issue not not parsing index nodes
 within snippets

---
 failedTests.txt                 | 32 +-------------------------------
 javascript/parseXmlJson.js      |  9 +++++++++
 javascript/searchRewriteTest.js |  9 +++++----
 3 files changed, 15 insertions(+), 35 deletions(-)

diff --git a/failedTests.txt b/failedTests.txt
index e866c2a57..736473b19 100644
--- a/failedTests.txt
+++ b/failedTests.txt
@@ -1,32 +1,2 @@
-abs: result is 0, expected occuer number is: 1
-accelerated_sequence: result is null, expected occuer number is: 1
-accumulate: result is 4, expected occuer number is: 3
-accumulate_n: result is null, expected occuer number is: 1
-accumulator: result is 3, expected occuer number is: 2
 Ackermann’s function: result is null, expected occuer number is: 1
-actual_value: result is null, expected occuer number is: 1
-Ada: result is 2, expected occuer number is: 1
-add (generic): result is 2, expected occuer number is: 1
-add_action: result is 1, expected occuer number is: 2
-add_complex: result is null, expected occuer number is: 1
-add_complex_to_javascript_num: result is null, expected occuer number is: 1
-addend: result is null, expected occuer number is: 1
-adder (primitive constraint): result is null, expected occuer number is: 1
-adder: result is 4, expected occuer number is: 3
-add_interval: result is null, expected occuer number is: 1
-add_lists: result is null, expected occuer number is: 1
-add_poly: result is null, expected occuer number is: 1
-add_rat: result is null, expected occuer number is: 1
-add_rule_or_assertion: result is null, expected occuer number is: 1
-add_streams: result is null, expected occuer number is: 1
-add_terms: result is null, expected occuer number is: 1
-add_to_agenda: result is 1, expected occuer number is: 2
-add_vect: result is 2, expected occuer number is: 1
-adjoin_arg: result is null, expected occuer number is: 1
-adjoin_set: result is 3, expected occuer number is: 5
-adjoin_term: result is 1, expected occuer number is: 2
-advance_pc: result is null, expected occuer number is: 1
-after_delay: result is 1, expected occuer number is: 2
-A’h-mose: result is null, expected occuer number is: 1
-algebra expression: result is null, expected occuer number is: 7
-algebraci specification for data: result is null, expected occuer number is: 1
\ No newline at end of file
+A’h-mose: result is null, expected occuer number is: 1
\ No newline at end of file
diff --git a/javascript/parseXmlJson.js b/javascript/parseXmlJson.js
index 35a9cad49..765c3af1e 100644
--- a/javascript/parseXmlJson.js
+++ b/javascript/parseXmlJson.js
@@ -340,6 +340,8 @@ const processTextFunctions = {
   },
 
   SNIPPET: (node, obj) => {
+    const indexNodes = node.getElementsByTagName("INDEX");
+
     if (node.getAttribute("HIDE") == "yes") {
       return;
     } else if (node.getAttribute("LATEX") == "yes") {
@@ -379,6 +381,10 @@ const processTextFunctions = {
           obj["body"] = obj["body"].replace(matchStr, newStr);
         }
       }
+  
+      for (let i = 0; i < indexNodes.length; i++) {
+        processTextJson(indexNodes[i], {});
+      }
 
       return;
     }
@@ -388,6 +394,9 @@ const processTextFunctions = {
     obj["latex"] = false;
     obj["id"] = snippet_count;
     processSnippetJson(node, obj);
+    for (let i = 0; i < indexNodes.length; i++) {
+      processTextJson(indexNodes[i], {});
+    }
   },
 
   SUBINDEX: (node, obj) => {
diff --git a/javascript/searchRewriteTest.js b/javascript/searchRewriteTest.js
index 789ad4ebd..483091057 100644
--- a/javascript/searchRewriteTest.js
+++ b/javascript/searchRewriteTest.js
@@ -213,8 +213,8 @@ const indexSearchTestCase = {
     "advance_pc": 1,
     "after_delay": 2,
     "A’h-mose": 1,
-    "algebra expression": 7,
-    "algebraci specification for data": 1,
+    "algebraic expression": 7,
+    "algebraic specification for data": 1,
 }
 
 const failedTests = [];
@@ -226,16 +226,17 @@ const writeFailureMessage = (key, searchResult) => {
 export function testIndexSearch() {
     for (const [key, value] of Object.entries(indexSearchTestCase)) {
         const result = search(key, indexTrie);
-        console.log(result);
+        //console.log(result);
         if (result === null) {
             writeFailureMessage(key, "null");
             continue;
         }
-        if (result.length !== value) {
+        if (result.length < value) {
             writeFailureMessage(key, result.length);
             continue;
         }
     }
+    
     fs.writeFileSync("failedTests.txt", failedTests.join("\n"));
 }
 

From e9a53df7402d51411730c4d0dd02585057801fcc Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 15:29:06 +0800
Subject: [PATCH 05/11] some cleaning

---
 failedTests.txt     | 2 --
 javascript/index.js | 3 ++-
 2 files changed, 2 insertions(+), 3 deletions(-)
 delete mode 100644 failedTests.txt

diff --git a/failedTests.txt b/failedTests.txt
deleted file mode 100644
index 736473b19..000000000
--- a/failedTests.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-Ackermann’s function: result is null, expected occuer number is: 1
-A’h-mose: result is null, expected occuer number is: 1
\ No newline at end of file
diff --git a/javascript/index.js b/javascript/index.js
index 56af1c035..24f568ebb 100644
--- a/javascript/index.js
+++ b/javascript/index.js
@@ -364,7 +364,8 @@ async function main() {
 
     await recursiveXmlToHtmlInOrder("parseXml");
     writeRewritedSearchData();
-    testIndexSearch();
+    // this is meant to be temp; also, will remove the original "generateSearchData" after the updation at the frontend is completed.
+    //testIndexSearch();
   }
 }
 

From 7910e230e8f5565449b8398401819c5a97b5c906 Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 15:43:40 +0800
Subject: [PATCH 06/11] fixed all the errors indicated in the failedTest.txt

---
 failedTests.txt                 |  0
 javascript/index.js             |  2 +-
 javascript/parseXmlJson.js      |  2 +-
 javascript/searchRewriteTest.js | 14 ++++++++++----
 urls.txt                        |  1 +
 5 files changed, 13 insertions(+), 6 deletions(-)
 create mode 100644 failedTests.txt
 create mode 100644 urls.txt

diff --git a/failedTests.txt b/failedTests.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/javascript/index.js b/javascript/index.js
index 24f568ebb..32ebd6b74 100644
--- a/javascript/index.js
+++ b/javascript/index.js
@@ -365,7 +365,7 @@ async function main() {
     await recursiveXmlToHtmlInOrder("parseXml");
     writeRewritedSearchData();
     // this is meant to be temp; also, will remove the original "generateSearchData" after the updation at the frontend is completed.
-    //testIndexSearch();
+    testIndexSearch();
   }
 }
 
diff --git a/javascript/parseXmlJson.js b/javascript/parseXmlJson.js
index 765c3af1e..d105e5336 100644
--- a/javascript/parseXmlJson.js
+++ b/javascript/parseXmlJson.js
@@ -131,7 +131,7 @@ const tagsWithIds = {
   SECTION: () => `#h${heading_count}`,
   FOOTNOTE: () => `#footnote-link-${footnote_count}`,
   DISPLAYFOOTNOTE: () => `#footnote-${display_footnote_count}`,
-  SNIPPET: () => `{snippet_count}`,
+  SNIPPET: () => `${snippet_count}`,
   //todo, fix this
   EXERCISE: () => `#ex-1.${exercise_count}`,
   DISPLAYFOOTNOTE: () => `#footnote-${display_footnote_count}`,
diff --git a/javascript/searchRewriteTest.js b/javascript/searchRewriteTest.js
index 483091057..f5a0a3256 100644
--- a/javascript/searchRewriteTest.js
+++ b/javascript/searchRewriteTest.js
@@ -3,6 +3,7 @@
 
 
 /* known issues: 
+what is the mechanism for link to snippet and latex? propably need to modify the frontend
 did not process the "seexml" file, so no see also in index
 did not process the latex, roman, italic, etc.
 */
@@ -164,7 +165,7 @@ averager (constraint), 261 (ex. 3.33)
 
 // I include 2 out of the 5 columns of all the index with A here.
 
-import { indexTrie, search } from "./searchRewrite"
+import { indexTrie, search, getUrl, autoComplete } from "./searchRewrite"
 import fs from "fs";
 
 const indexSearchTestCase = {
@@ -179,7 +180,7 @@ const indexSearchTestCase = {
     "accumulate": 3,
     "accumulate_n": 1,
     "accumulator": 2,
-    "Ackermann’s function": 1,
+    "Ackermann's function": 1,
     "acquire a mutex": 1,
     "actions, in register machine": 2,
     "actual_value": 1,
@@ -212,12 +213,13 @@ const indexSearchTestCase = {
     "administrative assistant, importance of": 1,
     "advance_pc": 1,
     "after_delay": 2,
-    "A’h-mose": 1,
+    "A'h-mose": 1,
     "algebraic expression": 7,
     "algebraic specification for data": 1,
 }
 
 const failedTests = [];
+const urls = {};
 const writeFailureMessage = (key, searchResult) => {
     failedTests.push(`${key}: result is ${searchResult}, expected occuer number is: ${indexSearchTestCase[key]}`);
 }
@@ -231,12 +233,16 @@ export function testIndexSearch() {
             writeFailureMessage(key, "null");
             continue;
         }
+        
+        urls[key] = result.map(getUrl);
+
         if (result.length < value) {
             writeFailureMessage(key, result.length);
             continue;
         }
     }
-    
+
     fs.writeFileSync("failedTests.txt", failedTests.join("\n"));
+    fs.writeFileSync("urls.txt", JSON.stringify(urls));
 }
 
diff --git a/urls.txt b/urls.txt
new file mode 100644
index 000000000..ed9eacacf
--- /dev/null
+++ b/urls.txt
@@ -0,0 +1 @@
+{"abs":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.61","https://sourceacademy.nus.edu.sg/sicpjs/1.1.62"],"absolute value":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.6#p1"],"abstract data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1#p2"],"abstraction":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.8#p3","https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#p3","https://sourceacademy.nus.edu.sg/sicpjs/4#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.3.2#p1","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined"],"abstraction barriers":["https://sourceacademy.nus.edu.sg/sicpjs/2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.1.2#p1","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p1","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p6","https://sourceacademy.nus.edu.sg/sicpjs/2.5#p2","https://sourceacademy.nus.edu.sg/sicpjs/4.1.2#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p34"],"abstract models for data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.3#footnote-1"],"abstract syntax":["https://sourceacademy.nus.edu.sg/sicpjs/4.1.1#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p2"],"accelerated_sequence":["https://sourceacademy.nus.edu.sg/sicpjs/3.5.38"],"accumulate":["https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#ex-1.4","https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#ex-1.5","https://sourceacademy.nus.edu.sg/sicpjs/2.2.36","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-1.6","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-1.11"],"accumulate_n":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.325","https://sourceacademy.nus.edu.sg/sicpjs/2.2.363"],"accumulator":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#p4","https://sourceacademy.nus.edu.sg/sicpjs/3.1.1#ex-1.1","https://sourceacademy.nus.edu.sg/sicpjs/3.1.1#ex-1.4"],"Ackermann's function":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.1#ex-1.2","https://sourceacademy.nus.edu.sg/sicpjs/1.2.1#ex-1.2"],"acquire a mutex":["https://sourceacademy.nus.edu.sg/sicpjs/3.4.2#p16"],"actions, in register machine":["https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined"],"actual_value":["https://sourceacademy.nus.edu.sg/sicpjs/4.2.22"],"Ada":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#ex-1.9","https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#ex-1.9"],"Adams, Norman I., IV":["https://sourceacademy.nus.edu.sg/sicpjs/4.1.7#footnote-1"],"add (generic)":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.11","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p10","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p12"],"add_action":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p8","https://sourceacademy.nus.edu.sg/sicpjs/3.3.413"],"add_complex":["https://sourceacademy.nus.edu.sg/sicpjs/2.4.13"],"add_complex_to_javascript_num":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.21"],"addend":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.26"],"adder (primitive constraint)":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.58"],"adder":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p3","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p6","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#ex-1.3","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#ex-1.5"],"add_interval":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.41"],"additivity":["https://sourceacademy.nus.edu.sg/sicpjs/2#p10","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p4","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/2.5.1#p6"],"add_lists":["https://sourceacademy.nus.edu.sg/sicpjs/4.2.32"],"add_poly":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.31"],"add_rat":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.11"],"address":["https://sourceacademy.nus.edu.sg/sicpjs/5.3.1#p1"],"address arithmetic":["https://sourceacademy.nus.edu.sg/sicpjs/5.3.1#p1"],"add_rule_or_assertion":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.424"],"add_streams":["https://sourceacademy.nus.edu.sg/sicpjs/3.5.210"],"add_terms":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.32"],"add_to_agenda":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p15","https://sourceacademy.nus.edu.sg/sicpjs/3.3.429"],"add_vect":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.4#ex-1.3","https://sourceacademy.nus.edu.sg/sicpjs/2.2.4#ex-1.9"],"adjoin_arg":["https://sourceacademy.nus.edu.sg/sicpjs/5.4.214"],"adjoining to a list with pair":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.1#p7"],"adjoin_set":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#p2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.32","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#ex-1.3","https://sourceacademy.nus.edu.sg/sicpjs/2.3.39","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#ex-1.8","https://sourceacademy.nus.edu.sg/sicpjs/2.3.45"],"adjoin_term":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p9","https://sourceacademy.nus.edu.sg/sicpjs/2.5.34"],"Adleman, Leonard":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.6#footnote-5"],"administrative assistant, importance of":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#p7"],"advance_pc":["https://sourceacademy.nus.edu.sg/sicpjs/5.2.35"],"after_delay":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p8","https://sourceacademy.nus.edu.sg/sicpjs/3.3.414"],"A'h-mose":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.4#footnote-4"],"algebraic expression":["https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p7","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p1"],"algebraic specification for data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.3#footnote-1"]}
\ No newline at end of file

From e88898ce445b5b19271dfd4ca1c03b3cd42fc960 Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 16:41:51 +0800
Subject: [PATCH 07/11] support urls to exercise

---
 javascript/parseXmlJson.js                    | 17 +++++++++++------
 .../processExerciseJson.js                    | 19 +++++++++++++++++++
 javascript/searchRewrite.js                   |  2 +-
 javascript/searchRewriteTest.js               | 10 +++++++---
 urls.txt                                      |  2 +-
 5 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/javascript/parseXmlJson.js b/javascript/parseXmlJson.js
index d105e5336..0355d53c1 100644
--- a/javascript/parseXmlJson.js
+++ b/javascript/parseXmlJson.js
@@ -12,6 +12,8 @@ import {
   recursivelyProcessTextSnippetJson
 } from "./processingFunctions";
 
+import {getIdForExerciseJson} from "./processingFunctions/processExerciseJson";
+
 import {
   generateSearchData
 } from "./generateSearchData";
@@ -122,18 +124,19 @@ const processLatex = (node, obj, inline) => {
   obj["body"] = math;
   obj["tag"] = "LATEX";
 };
-
+let latest_exercise_json_id = undefined;
 const tagsWithIds = {
-  TITLE: () => subsubsection_count>0? `#sec${chapterIndex}.${subsubsection_count}` :"",
+  "#document": () => "",
+  SUBSUBSECTION: () => subsubsection_count>0? `#sec${chapterIndex}.${subsubsection_count}` :"",
   TEXT:() => "#p" + paragraph_count,
   SUBHEADING: () => `#h${heading_count}`,
   SUBSUBHEADING: () => `#h${heading_count}`,
   SECTION: () => `#h${heading_count}`,
   FOOTNOTE: () => `#footnote-link-${footnote_count}`,
   DISPLAYFOOTNOTE: () => `#footnote-${display_footnote_count}`,
-  SNIPPET: () => `${snippet_count}`,
-  //todo, fix this
-  EXERCISE: () => `#ex-1.${exercise_count}`,
+  //SNIPPET: () => `${snippet_count}`,
+  
+  EXERCISE: () => latest_exercise_json_id,
   DISPLAYFOOTNOTE: () => `#footnote-${display_footnote_count}`,
 };
 const findParentID = (node) => {
@@ -211,6 +214,7 @@ const processTextFunctions = {
   },
 
   EXERCISE: (node, obj) => {
+    latest_exercise_json_id = getIdForExerciseJson(node);
     exercise_count += 1;
     processExerciseJson(node, obj, chapArrIndex, exercise_count);
   },
@@ -559,7 +563,8 @@ export const parseXmlJson = (doc, arr, filename) => {
   } else {
     displayTitle = chapterIndex + "\u00A0\u00A0" + chapterTitle;
   }
-
+  
+  latest_exercise_json_id = undefined;
   paragraph_count = 0;
   footnote_count = 0;
   display_footnote_count = 0;
diff --git a/javascript/processingFunctions/processExerciseJson.js b/javascript/processingFunctions/processExerciseJson.js
index 42c96af75..29712842d 100755
--- a/javascript/processingFunctions/processExerciseJson.js
+++ b/javascript/processingFunctions/processExerciseJson.js
@@ -35,4 +35,23 @@ const processExerciseJson = (node, obj) => {
   }
 };
 
+export const getIdForExerciseJson = (node) => {
+  const label = node.getElementsByTagName("LABEL")[0];
+  let labelName = "";
+
+  if (!label) {
+    labelName = "ex:unlabeled" + unlabeledEx;
+  } else {
+    labelName = label.getAttribute("NAME");
+  }
+
+  if (!referenceStore[labelName]) {
+    missingExerciseWarning(labelName);
+    return undefined;
+  }
+   
+  const displayName = referenceStore[labelName].displayName;
+  return `#ex-${displayName}`;
+}
+
 export default processExerciseJson;
diff --git a/javascript/searchRewrite.js b/javascript/searchRewrite.js
index 9270faf9c..760fbd48d 100644
--- a/javascript/searchRewrite.js
+++ b/javascript/searchRewrite.js
@@ -270,7 +270,7 @@ export const parseAndInsertToIdToContentMap = (json,chapterIndex, idStack = [])
         return;
     }
 
-    if(json.id) {
+    if(json.id && json.tag !== "SNIPPET") {
         const id = chapterIndex + json.id;
         idStack.push(id);
         idToContentMap[id] = "";
diff --git a/javascript/searchRewriteTest.js b/javascript/searchRewriteTest.js
index f5a0a3256..7814eec99 100644
--- a/javascript/searchRewriteTest.js
+++ b/javascript/searchRewriteTest.js
@@ -2,8 +2,10 @@
 // this file is meant to be temporary
 
 
-/* known issues: 
+/* todos and issues: 
+have not test urls, and urls for exercise can be impoved, currently point to the closest paragraph
 what is the mechanism for link to snippet and latex? propably need to modify the frontend
+works fine for things like <= (numeric comparison operator), but not for || (logical disjunction); could not test for " (double quote)
 did not process the "seexml" file, so no see also in index
 did not process the latex, roman, italic, etc.
 */
@@ -224,7 +226,6 @@ const writeFailureMessage = (key, searchResult) => {
     failedTests.push(`${key}: result is ${searchResult}, expected occuer number is: ${indexSearchTestCase[key]}`);
 }
 
-
 export function testIndexSearch() {
     for (const [key, value] of Object.entries(indexSearchTestCase)) {
         const result = search(key, indexTrie);
@@ -239,8 +240,11 @@ export function testIndexSearch() {
         if (result.length < value) {
             writeFailureMessage(key, result.length);
             continue;
-        }
+        }  
     }
+    
+    console.log(autoComplete("||", indexTrie));
+    console.log(search("|| (logical disjunction)", indexTrie));
 
     fs.writeFileSync("failedTests.txt", failedTests.join("\n"));
     fs.writeFileSync("urls.txt", JSON.stringify(urls));
diff --git a/urls.txt b/urls.txt
index ed9eacacf..ae5bb4227 100644
--- a/urls.txt
+++ b/urls.txt
@@ -1 +1 @@
-{"abs":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.61","https://sourceacademy.nus.edu.sg/sicpjs/1.1.62"],"absolute value":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.6#p1"],"abstract data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1#p2"],"abstraction":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.8#p3","https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#p3","https://sourceacademy.nus.edu.sg/sicpjs/4#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.3.2#p1","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined"],"abstraction barriers":["https://sourceacademy.nus.edu.sg/sicpjs/2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.1.2#p1","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p1","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p6","https://sourceacademy.nus.edu.sg/sicpjs/2.5#p2","https://sourceacademy.nus.edu.sg/sicpjs/4.1.2#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p34"],"abstract models for data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.3#footnote-1"],"abstract syntax":["https://sourceacademy.nus.edu.sg/sicpjs/4.1.1#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p2"],"accelerated_sequence":["https://sourceacademy.nus.edu.sg/sicpjs/3.5.38"],"accumulate":["https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#ex-1.4","https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#ex-1.5","https://sourceacademy.nus.edu.sg/sicpjs/2.2.36","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-1.6","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-1.11"],"accumulate_n":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.325","https://sourceacademy.nus.edu.sg/sicpjs/2.2.363"],"accumulator":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#p4","https://sourceacademy.nus.edu.sg/sicpjs/3.1.1#ex-1.1","https://sourceacademy.nus.edu.sg/sicpjs/3.1.1#ex-1.4"],"Ackermann's function":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.1#ex-1.2","https://sourceacademy.nus.edu.sg/sicpjs/1.2.1#ex-1.2"],"acquire a mutex":["https://sourceacademy.nus.edu.sg/sicpjs/3.4.2#p16"],"actions, in register machine":["https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined"],"actual_value":["https://sourceacademy.nus.edu.sg/sicpjs/4.2.22"],"Ada":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#ex-1.9","https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#ex-1.9"],"Adams, Norman I., IV":["https://sourceacademy.nus.edu.sg/sicpjs/4.1.7#footnote-1"],"add (generic)":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.11","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p10","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p12"],"add_action":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p8","https://sourceacademy.nus.edu.sg/sicpjs/3.3.413"],"add_complex":["https://sourceacademy.nus.edu.sg/sicpjs/2.4.13"],"add_complex_to_javascript_num":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.21"],"addend":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.26"],"adder (primitive constraint)":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.58"],"adder":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p3","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p6","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#ex-1.3","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#ex-1.5"],"add_interval":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.41"],"additivity":["https://sourceacademy.nus.edu.sg/sicpjs/2#p10","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p4","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/2.5.1#p6"],"add_lists":["https://sourceacademy.nus.edu.sg/sicpjs/4.2.32"],"add_poly":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.31"],"add_rat":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.11"],"address":["https://sourceacademy.nus.edu.sg/sicpjs/5.3.1#p1"],"address arithmetic":["https://sourceacademy.nus.edu.sg/sicpjs/5.3.1#p1"],"add_rule_or_assertion":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.424"],"add_streams":["https://sourceacademy.nus.edu.sg/sicpjs/3.5.210"],"add_terms":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.32"],"add_to_agenda":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p15","https://sourceacademy.nus.edu.sg/sicpjs/3.3.429"],"add_vect":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.4#ex-1.3","https://sourceacademy.nus.edu.sg/sicpjs/2.2.4#ex-1.9"],"adjoin_arg":["https://sourceacademy.nus.edu.sg/sicpjs/5.4.214"],"adjoining to a list with pair":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.1#p7"],"adjoin_set":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#p2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.32","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#ex-1.3","https://sourceacademy.nus.edu.sg/sicpjs/2.3.39","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#ex-1.8","https://sourceacademy.nus.edu.sg/sicpjs/2.3.45"],"adjoin_term":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p9","https://sourceacademy.nus.edu.sg/sicpjs/2.5.34"],"Adleman, Leonard":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.6#footnote-5"],"administrative assistant, importance of":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#p7"],"advance_pc":["https://sourceacademy.nus.edu.sg/sicpjs/5.2.35"],"after_delay":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p8","https://sourceacademy.nus.edu.sg/sicpjs/3.3.414"],"A'h-mose":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.4#footnote-4"],"algebraic expression":["https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p7","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/undefined","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p1"],"algebraic specification for data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.3#footnote-1"]}
\ No newline at end of file
+{"abs":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.6#p1","https://sourceacademy.nus.edu.sg/sicpjs/1.1.6#p4"],"absolute value":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.6#p1"],"abstract data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1#p2"],"abstraction":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.8#p3","https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#p3","https://sourceacademy.nus.edu.sg/sicpjs/4#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.3.2#p1","https://sourceacademy.nus.edu.sg/sicpjs/5.1.2","https://sourceacademy.nus.edu.sg/sicpjs/5.1.2"],"abstraction barriers":["https://sourceacademy.nus.edu.sg/sicpjs/2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.1.2#p1","https://sourceacademy.nus.edu.sg/sicpjs/2.1.2","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p1","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p6","https://sourceacademy.nus.edu.sg/sicpjs/2.5#p2","https://sourceacademy.nus.edu.sg/sicpjs/4.1.2#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p34"],"abstract models for data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.3#footnote-1"],"abstract syntax":["https://sourceacademy.nus.edu.sg/sicpjs/4.1.1#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p2"],"accelerated_sequence":["https://sourceacademy.nus.edu.sg/sicpjs/3.5.3#p9"],"accumulate":["https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#ex-1.32","https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#ex-1.33","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#p8","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-2.38","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-2.43"],"accumulate_n":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-2.36","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-2.43"],"accumulator":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#p4","https://sourceacademy.nus.edu.sg/sicpjs/3.1.1#ex-3.1","https://sourceacademy.nus.edu.sg/sicpjs/3.1.1#ex-3.4"],"Ackermann's function":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.1#ex-1.10","https://sourceacademy.nus.edu.sg/sicpjs/1.2.1#ex-1.10"],"acquire a mutex":["https://sourceacademy.nus.edu.sg/sicpjs/3.4.2#p16"],"actions, in register machine":["https://sourceacademy.nus.edu.sg/sicpjs/5.1.1","https://sourceacademy.nus.edu.sg/sicpjs/5.1.1"],"actual_value":["https://sourceacademy.nus.edu.sg/sicpjs/4.2.2#p6"],"Ada":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#ex-4.61","https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#ex-4.61"],"Adams, Norman I., IV":["https://sourceacademy.nus.edu.sg/sicpjs/4.1.7#footnote-1"],"add (generic)":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.1#p2","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p10","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p12"],"add_action":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p8","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p12"],"add_complex":["https://sourceacademy.nus.edu.sg/sicpjs/2.4.1#p6"],"add_complex_to_javascript_num":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.2#p2"],"addend":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p6"],"adder (primitive constraint)":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.5#p11"],"adder":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p3","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p6","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#ex-3.30","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#ex-3.32"],"add_interval":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.4#p4"],"additivity":["https://sourceacademy.nus.edu.sg/sicpjs/2#p10","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p4","https://sourceacademy.nus.edu.sg/sicpjs/2.4.3","https://sourceacademy.nus.edu.sg/sicpjs/2.4.3","https://sourceacademy.nus.edu.sg/sicpjs/2.5.1#p6"],"add_lists":["https://sourceacademy.nus.edu.sg/sicpjs/4.2.3#p3"],"add_poly":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p6"],"add_rat":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.1#p4"],"address":["https://sourceacademy.nus.edu.sg/sicpjs/5.3.1#p1"],"address arithmetic":["https://sourceacademy.nus.edu.sg/sicpjs/5.3.1#p1"],"add_rule_or_assertion":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p27"],"add_streams":["https://sourceacademy.nus.edu.sg/sicpjs/3.5.2#p7"],"add_terms":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p10"],"add_to_agenda":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p15","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p24"],"add_vect":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.4#ex-2.46","https://sourceacademy.nus.edu.sg/sicpjs/2.2.4#ex-2.52"],"adjoin_arg":["https://sourceacademy.nus.edu.sg/sicpjs/5.4.2#footnote-3"],"adjoining to a list with pair":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.1#p7"],"adjoin_set":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#p2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#p3","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#ex-2.61","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#p12","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#ex-2.66","https://sourceacademy.nus.edu.sg/sicpjs/2.3.4#p14"],"adjoin_term":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p9","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p16"],"Adleman, Leonard":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.6#footnote-5"],"administrative assistant, importance of":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#p7"],"advance_pc":["https://sourceacademy.nus.edu.sg/sicpjs/5.2.3#p4"],"after_delay":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p8","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p16"],"A'h-mose":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.4#footnote-4"],"algebraic expression":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p1"],"algebraic specification for data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.3#footnote-1"]}
\ No newline at end of file

From 81497cd3ee69c17f9aef5d60a7df2c516fe11741 Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 16:46:59 +0800
Subject: [PATCH 08/11] commented out testcode, deleted temp files

---
 failedTests.txt                 | 0
 javascript/index.js             | 2 +-
 javascript/searchRewriteTest.js | 4 ++--
 urls.txt                        | 1 -
 4 files changed, 3 insertions(+), 4 deletions(-)
 delete mode 100644 failedTests.txt
 delete mode 100644 urls.txt

diff --git a/failedTests.txt b/failedTests.txt
deleted file mode 100644
index e69de29bb..000000000
diff --git a/javascript/index.js b/javascript/index.js
index 32ebd6b74..24f568ebb 100644
--- a/javascript/index.js
+++ b/javascript/index.js
@@ -365,7 +365,7 @@ async function main() {
     await recursiveXmlToHtmlInOrder("parseXml");
     writeRewritedSearchData();
     // this is meant to be temp; also, will remove the original "generateSearchData" after the updation at the frontend is completed.
-    testIndexSearch();
+    //testIndexSearch();
   }
 }
 
diff --git a/javascript/searchRewriteTest.js b/javascript/searchRewriteTest.js
index 7814eec99..94e86f573 100644
--- a/javascript/searchRewriteTest.js
+++ b/javascript/searchRewriteTest.js
@@ -3,8 +3,7 @@
 
 
 /* todos and issues: 
-have not test urls, and urls for exercise can be impoved, currently point to the closest paragraph
-what is the mechanism for link to snippet and latex? propably need to modify the frontend
+ propably need to modify the frontend to enable link to snippets
 works fine for things like <= (numeric comparison operator), but not for || (logical disjunction); could not test for " (double quote)
 did not process the "seexml" file, so no see also in index
 did not process the latex, roman, italic, etc.
@@ -167,6 +166,7 @@ averager (constraint), 261 (ex. 3.33)
 
 // I include 2 out of the 5 columns of all the index with A here.
 
+// manually tested the exercise urls to accumulate and accumulate_n, they are correct
 import { indexTrie, search, getUrl, autoComplete } from "./searchRewrite"
 import fs from "fs";
 
diff --git a/urls.txt b/urls.txt
deleted file mode 100644
index ae5bb4227..000000000
--- a/urls.txt
+++ /dev/null
@@ -1 +0,0 @@
-{"abs":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.6#p1","https://sourceacademy.nus.edu.sg/sicpjs/1.1.6#p4"],"absolute value":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.6#p1"],"abstract data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1#p2"],"abstraction":["https://sourceacademy.nus.edu.sg/sicpjs/1.1.8#p3","https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#p3","https://sourceacademy.nus.edu.sg/sicpjs/4#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.3.2#p1","https://sourceacademy.nus.edu.sg/sicpjs/5.1.2","https://sourceacademy.nus.edu.sg/sicpjs/5.1.2"],"abstraction barriers":["https://sourceacademy.nus.edu.sg/sicpjs/2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.1.2#p1","https://sourceacademy.nus.edu.sg/sicpjs/2.1.2","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p1","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p6","https://sourceacademy.nus.edu.sg/sicpjs/2.5#p2","https://sourceacademy.nus.edu.sg/sicpjs/4.1.2#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p34"],"abstract models for data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.3#footnote-1"],"abstract syntax":["https://sourceacademy.nus.edu.sg/sicpjs/4.1.1#p3","https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p2"],"accelerated_sequence":["https://sourceacademy.nus.edu.sg/sicpjs/3.5.3#p9"],"accumulate":["https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#ex-1.32","https://sourceacademy.nus.edu.sg/sicpjs/1.3.1#ex-1.33","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#p8","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-2.38","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-2.43"],"accumulate_n":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-2.36","https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#ex-2.43"],"accumulator":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.3#p4","https://sourceacademy.nus.edu.sg/sicpjs/3.1.1#ex-3.1","https://sourceacademy.nus.edu.sg/sicpjs/3.1.1#ex-3.4"],"Ackermann's function":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.1#ex-1.10","https://sourceacademy.nus.edu.sg/sicpjs/1.2.1#ex-1.10"],"acquire a mutex":["https://sourceacademy.nus.edu.sg/sicpjs/3.4.2#p16"],"actions, in register machine":["https://sourceacademy.nus.edu.sg/sicpjs/5.1.1","https://sourceacademy.nus.edu.sg/sicpjs/5.1.1"],"actual_value":["https://sourceacademy.nus.edu.sg/sicpjs/4.2.2#p6"],"Ada":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#ex-4.61","https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#ex-4.61"],"Adams, Norman I., IV":["https://sourceacademy.nus.edu.sg/sicpjs/4.1.7#footnote-1"],"add (generic)":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.1#p2","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p10","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p12"],"add_action":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p8","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p12"],"add_complex":["https://sourceacademy.nus.edu.sg/sicpjs/2.4.1#p6"],"add_complex_to_javascript_num":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.2#p2"],"addend":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p6"],"adder (primitive constraint)":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.5#p11"],"adder":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p3","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p6","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#ex-3.30","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#ex-3.32"],"add_interval":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.4#p4"],"additivity":["https://sourceacademy.nus.edu.sg/sicpjs/2#p10","https://sourceacademy.nus.edu.sg/sicpjs/2.4#p4","https://sourceacademy.nus.edu.sg/sicpjs/2.4.3","https://sourceacademy.nus.edu.sg/sicpjs/2.4.3","https://sourceacademy.nus.edu.sg/sicpjs/2.5.1#p6"],"add_lists":["https://sourceacademy.nus.edu.sg/sicpjs/4.2.3#p3"],"add_poly":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p6"],"add_rat":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.1#p4"],"address":["https://sourceacademy.nus.edu.sg/sicpjs/5.3.1#p1"],"address arithmetic":["https://sourceacademy.nus.edu.sg/sicpjs/5.3.1#p1"],"add_rule_or_assertion":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.4#p27"],"add_streams":["https://sourceacademy.nus.edu.sg/sicpjs/3.5.2#p7"],"add_terms":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p10"],"add_to_agenda":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p15","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p24"],"add_vect":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.4#ex-2.46","https://sourceacademy.nus.edu.sg/sicpjs/2.2.4#ex-2.52"],"adjoin_arg":["https://sourceacademy.nus.edu.sg/sicpjs/5.4.2#footnote-3"],"adjoining to a list with pair":["https://sourceacademy.nus.edu.sg/sicpjs/2.2.1#p7"],"adjoin_set":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#p2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#p3","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#ex-2.61","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#p12","https://sourceacademy.nus.edu.sg/sicpjs/2.3.3#ex-2.66","https://sourceacademy.nus.edu.sg/sicpjs/2.3.4#p14"],"adjoin_term":["https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p9","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p16"],"Adleman, Leonard":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.6#footnote-5"],"administrative assistant, importance of":["https://sourceacademy.nus.edu.sg/sicpjs/4.4.1#p7"],"advance_pc":["https://sourceacademy.nus.edu.sg/sicpjs/5.2.3#p4"],"after_delay":["https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p8","https://sourceacademy.nus.edu.sg/sicpjs/3.3.4#p16"],"A'h-mose":["https://sourceacademy.nus.edu.sg/sicpjs/1.2.4#footnote-4"],"algebraic expression":["https://sourceacademy.nus.edu.sg/sicpjs/2.3.2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2#p7","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2","https://sourceacademy.nus.edu.sg/sicpjs/2.3.2","https://sourceacademy.nus.edu.sg/sicpjs/2.5.3#p1"],"algebraic specification for data":["https://sourceacademy.nus.edu.sg/sicpjs/2.1.3#footnote-1"]}
\ No newline at end of file

From 00bdb27bd9926117ffbf5da8012e2a5c3418357c Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Sun, 1 Oct 2023 17:29:01 +0800
Subject: [PATCH 09/11] add test for corrrectness of urls, and run and passed
 the test

---
 javascript/searchRewriteTest.js | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/javascript/searchRewriteTest.js b/javascript/searchRewriteTest.js
index 94e86f573..e2a23ffd8 100644
--- a/javascript/searchRewriteTest.js
+++ b/javascript/searchRewriteTest.js
@@ -226,7 +226,7 @@ const writeFailureMessage = (key, searchResult) => {
     failedTests.push(`${key}: result is ${searchResult}, expected occuer number is: ${indexSearchTestCase[key]}`);
 }
 
-export function testIndexSearch() {
+export async function testIndexSearch() {
     for (const [key, value] of Object.entries(indexSearchTestCase)) {
         const result = search(key, indexTrie);
         //console.log(result);
@@ -246,6 +246,27 @@ export function testIndexSearch() {
     console.log(autoComplete("||", indexTrie));
     console.log(search("|| (logical disjunction)", indexTrie));
 
+    async function testURLs() {
+        console.log("Testing urls");
+        for (const [key, urlArray] of Object.entries(urls)) {
+            for (const url of urlArray) {
+                try {
+                  const response = await fetch(url);
+            
+                  if (!response.ok) {
+                    console.log(key + ": " + url + " is not working");
+                  }
+      
+                } catch (error) {
+                  console.error(key + ": " + url + " is not working");
+                }
+              }
+        }
+        console.log("Done testing urls");       
+    }
+      
+    await testURLs();
+
     fs.writeFileSync("failedTests.txt", failedTests.join("\n"));
     fs.writeFileSync("urls.txt", JSON.stringify(urls));
 }

From 07af486d4386217becc0f53cdf2527078ea216ee Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Wed, 18 Oct 2023 17:13:55 +0800
Subject: [PATCH 10/11] fix multiple bugs for text search

---
 javascript/searchRewrite.js | 103 ++++++++++++------------------------
 1 file changed, 34 insertions(+), 69 deletions(-)

diff --git a/javascript/searchRewrite.js b/javascript/searchRewrite.js
index 760fbd48d..cebd9fd46 100644
--- a/javascript/searchRewrite.js
+++ b/javascript/searchRewrite.js
@@ -79,62 +79,7 @@ const parseIndexSearchEntryTo = (node, json) => {
     }
 };
 
-// currently not used, a map of all the tags in json, good reference
-const processingFunctions = {
-    '#text': 1,
-  
-    B: 1,
-    BR: 1,
-  
-    DISPLAYFOOTNOTE: 1,
-  
-    EM: 1,
-  
-    EPIGRAPH: 1,
-  
-    EXERCISE: 1,
-  
-    FIGURE: 1,
-  
-    FOOTNOTE_REF: 1,
-  
-    JAVASCRIPTINLINE: 1,
-  
-    LATEX: 1,
-  
-    LI: 1,
-  
-    LINK: 1,
-  
-    META: 1,
-  
-    OL: 1,
-  
-    REF: 1,
-  
-    REFERENCE: 1,
-  
-    SNIPPET: 1,
-  
-    SUBHEADING: 1,
-  
-    SUBSUBHEADING: 1,
-  
-    TABLE: 1,
-  
-    TEXT: 1,
-  
-    TITLE:1,
-  
-    TT: 1,
-  
-    UL: 1,
-};
-// this list is dependent on the implementation of parseXmlJson and frontend
-// so why parsing json at frontend? why not send html to frontend which will then only need to care
-
 const indexParsers = {
-    // plain text nodes
      "#text": (node,json) => {
         json["text"] += node.nodeValue;
     },
@@ -259,41 +204,61 @@ const indexParsers = {
 
 export const parseAndInsertToIndexTrie = (node, json) => {
     parseIndexSearchEntryTo(node, json);
-    insert(json.text, json, indexTrie);
+    const frontEndDisplayable = {}
+    frontEndDisplayable["id"] = json.id;
+    // build text for front end display, build prefix, main text and subindex text seperately
+    let chapterId = json.id.split("#")[0];
+    const num = chapterId.split(".").length;
+    if(num === 1) {
+        chapterId = "    " + chapterId;
+    } else if (num === 2) {
+        chapterId = "  " + chapterId;
+    }
+    frontEndDisplayable["text"] = chapterId + ": " + json.text;
+    if(json.SUBINDEX) {
+        frontEndDisplayable["text"] += ` :: ${json.SUBINDEX.text}`;
+        if(json.SUBINDEX.ORDER) {
+            frontEndDisplayable["order"] = json.SUBINDEX.ORDER;
+        }
+    }
+
+    insert(json.text, frontEndDisplayable, indexTrie);
 }
 
-export const parseAndInsertToIdToContentMap = (json,chapterIndex, idStack = []) => {
+export const parseAndInsertToIdToContentMap = (json,chapterIndex, parentID =chapterIndex) => {
     if(Array.isArray(json)) {
         for (let i = 0; i < json.length; i++) {
-        parseAndInsertToIdToContentMap(json[i],chapterIndex,idStack);
+        parseAndInsertToIdToContentMap(json[i],chapterIndex,parentID);
         }
         return;
     }
 
     if(json.id && json.tag !== "SNIPPET") {
-        const id = chapterIndex + json.id;
-        idStack.push(id);
+        const id = json.id.includes(chapterIndex)? chapterIndex: chapterIndex + json.id;
+        parentID = id;
         idToContentMap[id] = "";
     }
     if(json.body) {
-        idToContentMap[idStack[idStack.length-1]] += json.body;
+        idToContentMap[parentID] += json.body;
     }
     if(json.child) {
-        parseAndInsertToIdToContentMap(json.child,chapterIndex,idStack);
-    }
-    if(json.id) {
-        idStack.pop();
+        parseAndInsertToIdToContentMap(json.child,chapterIndex,parentID);
     }
 }
 
 const buildTextTrie = () => {
+    console.log("enter buildTextTrie")
     for (const [key, value] of Object.entries(idToContentMap)) {
-        const words = value.trim().replace(/\s+/g," ").toLowerCase().replace(/\n/gi, "").split(" ").filter(a => a != "" && a != '\"');
-        for (const word of words) {
-            insert(word, key, textTrie);
+        const temp = value.match(/\b\w+\b/g);
+        if(temp === null) {
+            // some json node does not have text content, there id is stored, but no text value, so we skip them here
+            continue;
         }
+        const words = Array.from(new Set(temp.map(word => word.toLowerCase())));
+        words.map(word => insert(word, key, textTrie));
+        
+    }
     }
-}
 
 export const writeRewritedSearchData = () => {
     buildTextTrie();

From 5beee2ca73816c7db3a936e852a88f4fc4e9c022 Mon Sep 17 00:00:00 2001
From: yiwen101 <e0968880@u.nus.edu>
Date: Wed, 18 Oct 2023 22:38:22 +0800
Subject: [PATCH 11/11] support order, open close, and simplify the frontend's
 parsing job

---
 javascript/searchRewrite.js | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/javascript/searchRewrite.js b/javascript/searchRewrite.js
index cebd9fd46..a14706db4 100644
--- a/javascript/searchRewrite.js
+++ b/javascript/searchRewrite.js
@@ -202,9 +202,25 @@ const indexParsers = {
     },  
 };
 
+const maintainOpenClose = (json, writeTo) => {
+    if(json.OPEN) {
+        writeTo["text"] += " (begin of range)";
+        if(!writeTo["id"].includes("#")) {
+            writeTo["id"] += "#begin";
+        }
+    }
+
+    if(json.CLOSE) {
+        writeTo["text"] += " (end of range)";
+        if(!writeTo["id"].includes("#")) {
+            writeTo["id"] += "#end";
+        }
+    }
+}
+
 export const parseAndInsertToIndexTrie = (node, json) => {
     parseIndexSearchEntryTo(node, json);
-    const frontEndDisplayable = {}
+    const frontEndDisplayable = {text:"", order:"", id:"", hasSubindex: false};
     frontEndDisplayable["id"] = json.id;
     // build text for front end display, build prefix, main text and subindex text seperately
     let chapterId = json.id.split("#")[0];
@@ -216,12 +232,17 @@ export const parseAndInsertToIndexTrie = (node, json) => {
     }
     frontEndDisplayable["text"] = chapterId + ": " + json.text;
     if(json.SUBINDEX) {
+        frontEndDisplayable["hasSubindex"] = true;
         frontEndDisplayable["text"] += ` :: ${json.SUBINDEX.text}`;
         if(json.SUBINDEX.ORDER) {
             frontEndDisplayable["order"] = json.SUBINDEX.ORDER;
+        } else {
+            frontEndDisplayable["order"] = json.SUBINDEX.text;
         }
+        
+        maintainOpenClose(json.SUBINDEX, frontEndDisplayable);
     }
-
+    maintainOpenClose(json, frontEndDisplayable);
     insert(json.text, frontEndDisplayable, indexTrie);
 }