Feat: Setting to allow disabling stemming

src/index.ts

@@ -1,16 +1,22 @@
-import { Plugin } from 'obsidian';
+import {debounce, Plugin} from 'obsidian';
 import type { Extension } from '@codemirror/state';
 
 import Search from './search';
 import { PluginHelper } from './plugin-helper';
 import { Indexer } from './indexing/indexer';
 import { suggestionsExtension } from './cmExtension/suggestionsExtension';
+import {DEFAULT_SETTINGS, SidekickSettings} from "~/settings/sidekickSettings";
+import SicekickSettingsTab from "~/settings/sidekickSettingsTab";
 
 export default class TagsAutosuggestPlugin extends Plugin {
   private editorExtension: Extension[] = [];
+  settings: SidekickSettings;
+  private _rebuildIndex: CallableFunction;
 
   public async onload(): Promise<void> {
     console.log('Autosuggest plugin: loading plugin', new Date().toLocaleString());
+    this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
+    this.addSettingTab(new SicekickSettingsTab(this.app, this));
 
     const pluginHelper = new PluginHelper(this);
     const indexer = new Indexer(pluginHelper);
@@ -23,17 +29,18 @@ export default class TagsAutosuggestPlugin extends Plugin {
 
     // Re/load highlighting extension after any changes to index
     indexer.on('indexRebuilt', () => {
-      const search = new Search(indexer);
+      const search = new Search(indexer, this.settings);
       this.updateEditorExtension(suggestionsExtension(search, this.app));
     });
 
     indexer.on('indexUpdated', () => {
-      const search = new Search(indexer);
+      const search = new Search(indexer, this.settings);
       this.updateEditorExtension(suggestionsExtension(search, this.app));
     });
 
     // Build search index on startup (very expensive process)
     pluginHelper.onLayoutReady(() => indexer.buildIndex());
+    this._rebuildIndex = debounce(() => indexer.buildIndex(true), 5000, true);
   }
 
   /**
@@ -45,7 +52,18 @@ export default class TagsAutosuggestPlugin extends Plugin {
     this.app.workspace.updateOptions();
   }
 
+  public rebuildIndex() {
+    // This is very expensive, so debounce this
+    this._rebuildIndex();
+  }
+
   public async onunload(): Promise<void> {
     console.log('Autosuggest plugin: unloading plugin', new Date().toLocaleString());
   }
+
+  public async saveSettings(): Promise<void> {
+    await this.saveData(this.settings);
+    // TODO: There may be instances in the future where saving the settings does not require rebuilding the index.
+    this.rebuildIndex();
+  }
 }

src/indexing/indexer.ts

@@ -6,10 +6,11 @@ import type { TFile } from 'obsidian';
 import { stemPhrase } from '../stemmers';
 import { WordPermutationsTokenizer } from '../tokenizers';
 import type { PluginHelper } from '../plugin-helper';
+import {SidekickSettings} from "~/settings/sidekickSettings";
 
 type Document = {
   fileCreationTime: number;
-  type: 'tag' | 'alias' | 'page' | 'page-token';
+  type: 'tag' | 'alias' | 'page' | 'page-token' | 'unresolved';
   keyword: string;
   originalText: string;
   replaceText: string;
@@ -21,19 +22,29 @@ interface IndexerEvents {
 }
 
 export class Indexer extends TypedEmitter<IndexerEvents> {
+  private db: lokijs;
   private documents: Collection<Document>;
   private permutationTokenizer: WordPermutationsTokenizer;
+  private settings: SidekickSettings;
+  private keywordsFilter: Set<string>;
+
 
   constructor(private pluginHelper: PluginHelper) {
     super();
 
-    const db = new lokijs('sidekick');
+    this.db = new lokijs('sidekick');
 
-    this.documents = db.addCollection<Document>('documents', {
-      indices: ['fileCreationTime', 'keyword'],
-    });
+    this.createCollection();
 
     this.permutationTokenizer = new WordPermutationsTokenizer();
+    this.settings = pluginHelper.getSettings();
+
+  }
+
+  private createCollection() {
+    this.documents = this.db.addCollection<Document>('documents', {
+      indices: ['fileCreationTime', 'keyword'],
+    });
   }
 
   public getKeywords(): string[] {
@@ -53,7 +64,16 @@ export class Indexer extends TypedEmitter<IndexerEvents> {
     });
   }
 
-  public buildIndex(): void {
+  public buildIndex(rebuild=false): void {
+    this.keywordsFilter = new Set(this.settings.keywordsFilter
+      .replace(/\s/g,'')
+      .split(",")
+      .map(s => s.trim()));
+    if (rebuild) {
+      this.db.removeCollection("documents");
+      this.createCollection();
+      this.documents.removeWhere(() => true);
+    }
     this.pluginHelper.getAllFiles().forEach((file) => this.indexFile(file));
     this.emit('indexRebuilt');
   }
@@ -69,42 +89,77 @@ export class Indexer extends TypedEmitter<IndexerEvents> {
   }
 
   private indexFile(file: TFile): void {
-    this.documents.insert({
-      fileCreationTime: file.stat.ctime,
-      type: 'page',
-      keyword: stemPhrase(file.basename),
-      originalText: file.basename,
-      replaceText: `[[${file.basename}]]`,
-    });
-
-    this.permutationTokenizer.tokenize(file.basename).forEach((token) => {
+    // TODO Emile: not sure if should toLowerCase() here.
+    const baseNameKeyword = this.settings.enableStemming ? stemPhrase(file.basename) : file.basename.toLowerCase();
+    // Emile: Keywords filter is applied at every insertion, which is a bit of a code smell.
+    //  however, it cannot quickly be done in a post-processing step because that'd need a full iteration
+    //  on every call of replaceFileIndices()
+    if (!this.keywordsFilter.has(baseNameKeyword)) {
       this.documents.insert({
         fileCreationTime: file.stat.ctime,
-        type: 'page-token',
-        keyword: token,
+        type: 'page',
+        keyword: baseNameKeyword,
         originalText: file.basename,
         replaceText: `[[${file.basename}]]`,
       });
-    });
+    }
+
+    if (this.settings.enableStemming) {
+      this.permutationTokenizer.tokenize(file.basename).forEach((keyword) => {
+        if (!this.keywordsFilter.has(keyword)) {
+          this.documents.insert({
+            fileCreationTime: file.stat.ctime,
+            type: 'page-token',
+            keyword,
+            originalText: file.basename,
+            replaceText: `[[${file.basename}]]`,
+          });
+        }
+      });
+    }
 
     this.pluginHelper.getAliases(file).forEach((alias) => {
-      this.documents.insert({
-        fileCreationTime: file.stat.ctime,
-        type: 'alias',
-        keyword: alias.toLowerCase(),
-        originalText: file.basename,
-        replaceText: `[[${file.basename}|${alias}]]`,
-      });
+      const keyword = alias.toLowerCase();
+      if (!this.keywordsFilter.has(keyword)) {
+        this.documents.insert({
+          fileCreationTime: file.stat.ctime,
+          type: 'alias',
+          keyword,
+          originalText: file.basename,
+          replaceText: `[[${file.basename}|${alias}]]`,
+        });
+      }
     });
 
-    this.pluginHelper.getTags(file).forEach((tag) => {
-      this.documents.insert({
-        fileCreationTime: file.stat.ctime,
-        type: 'tag',
-        keyword: tag.replace(/#/, '').toLowerCase(),
-        originalText: tag,
-        replaceText: tag,
+    if (this.settings.matchTags) {
+      // TODO: This can probably be done more efficiently by iterating getAllTags.
+      this.pluginHelper.getTags(file).forEach((tag) => {
+        const keyword = tag.replace(/#/, '').toLowerCase();
+        if (!this.keywordsFilter.has(keyword)) {
+          this.documents.insert({
+            fileCreationTime: file.stat.ctime,
+            type: 'tag',
+            keyword,
+            originalText: tag,
+            replaceText: tag,
+          });
+        }
       });
-    });
+    }
+
+    if (this.settings.matchUnresolved) {
+      this.pluginHelper.getUnresolvedLinks(file).forEach((link) => {
+        const keyword = link.toLowerCase();
+        if (!this.keywordsFilter.has(keyword)) {
+          this.documents.insert( {
+            fileCreationTime: file.stat.ctime,
+            type: 'unresolved',
+            keyword,
+            originalText: link,
+            replaceText: `[[${link}]]`
+          });
+        }
+      })
+    }
   }
 }

src/plugin-helper.ts

@@ -1,10 +1,12 @@
 import _ from 'lodash';
-import { parseFrontMatterTags, TFile, Plugin } from 'obsidian';
+import { parseFrontMatterTags, TFile} from 'obsidian';
 
 import { getAliases } from './utils/getAliases';
+import {SidekickSettings} from "~/settings/sidekickSettings";
+import TagsAutosuggestPlugin from "~/index";
 
 export class PluginHelper {
-  constructor(private plugin: Plugin) {}
+  constructor(private plugin: TagsAutosuggestPlugin) {}
 
   public get activeFile(): TFile | undefined {
     return this.plugin.app.workspace.getActiveFile();
@@ -25,6 +27,12 @@ export class PluginHelper {
     return this.plugin.app.vault.getMarkdownFiles();
   }
 
+  public getUnresolvedLinks(file: TFile): string[] {
+    const unresolvedLinksDict = this.plugin.app.metadataCache.unresolvedLinks;
+    const record = unresolvedLinksDict[file.path];
+    return Object.keys(record);
+  }
+
   public onLayoutReady(callback: () => void): void {
     this.plugin.app.workspace.onLayoutReady(() => callback());
   }
@@ -52,4 +60,8 @@ export class PluginHelper {
       parseFrontMatterTags(this.plugin.app.metadataCache.getFileCache(file)?.frontmatter) ?? []
     );
   }
+
+  public getSettings(): SidekickSettings {
+    return this.plugin.settings;
+  }
 }

src/search/index.ts

@@ -4,9 +4,8 @@ import { Trie } from '@tanishiking/aho-corasick';
 import type { Indexer } from '../indexing/indexer';
 import { redactText } from './redactText';
 import { mapStemToOriginalText } from './mapStemToOriginalText';
-import { WordPunctStemTokenizer } from '../tokenizers';
-
-const tokenizer = new WordPunctStemTokenizer();
+import {SidekickTokenizer, WordPunctStemTokenizer, WordPunctTokenizer} from '../tokenizers';
+import {SidekickSettings} from "~/settings/sidekickSettings";
 
 export type SearchResult = {
   start: number;
@@ -21,8 +20,9 @@ const isEqual = (a: SearchResult, b: SearchResult) => {
 
 export default class Search {
   private trie: Trie;
+  private tokenizer: SidekickTokenizer;
 
-  constructor(private indexer: Indexer) {
+  constructor(private indexer: Indexer, private settings: SidekickSettings) {
     const keywords = this.indexer.getKeywords();
 
     // Generating the Trie is expensive, so we only do it once
@@ -31,6 +31,12 @@ export default class Search {
       onlyWholeWords: true,
       caseInsensitive: true,
     });
+    if (settings.enableStemming) {
+      this.tokenizer = new WordPunctStemTokenizer()
+    }
+    else {
+      this.tokenizer = new WordPunctTokenizer();
+    }
   }
 
   public getReplacementSuggestions(keyword: string): string[] {
@@ -40,17 +46,18 @@ export default class Search {
 
   public find(text: string): SearchResult[] {
     const redactedText = redactText(text); // Redact text that we don't want to be searched
+    const tokens = this.tokenizer.tokenize(redactedText);
 
     // Stem the text
-    const tokens = tokenizer.tokenize(redactedText);
-    const stemmedText = tokens.map((t) => t.stem).join('');
+    const searchText = tokens.map((t) => t.stem).join('');
 
-    // Search stemmed text
-    const emits = this.trie.parseText(stemmedText);
+    // Search (stemmed) text
+    const emits = this.trie.parseText(searchText);
 
     // Map stemmed results to original text
     return _.chain(emits)
       .map((emit) => mapStemToOriginalText(emit, tokens))
+      .filter((r) => r !== null)
       .uniqWith(isEqual)
       .filter((result) => this.keywordExistsInIndex(result.indexKeyword))
       .sort((a, b) => a.start - b.start) // Must sort by start position to prepare for highlighting

src/search/mapStemToOriginalText.ts

@@ -11,11 +11,16 @@ import { Token } from '../tokenizers';
  * @param tokens
  * @returns
  */
-export const mapStemToOriginalText = (searchResult: Emit, tokens: Token[]): SearchResult => {
+export const mapStemToOriginalText = (searchResult: Emit, tokens: Token[]): SearchResult | null => {
   const matchingTokens = tokens.filter(
     (token) => token.stemStart >= searchResult.start && token.stemEnd <= searchResult.end + 1
   );
 
+  if (matchingTokens.length === 0) {
+    // Examples where this is possible: an external url contains a key
+    return null;
+  }
+
   return {
     start: matchingTokens[0].originalStart,
     end: matchingTokens[matchingTokens.length - 1].originalEnd,

src/settings/sidekickSettings.ts

@@ -0,0 +1,13 @@
+export interface SidekickSettings{
+  enableStemming: boolean;
+  matchTags: boolean;
+  matchUnresolved: boolean;
+  keywordsFilter: string;
+}
+
+export const DEFAULT_SETTINGS: SidekickSettings = {
+  enableStemming: true,
+  matchTags: true,
+  matchUnresolved: true,
+  keywordsFilter: ""
+}