Desktop: Add support for OCR (#8975)

.eslintignore

@@ -753,6 +753,8 @@ packages/lib/services/database/addMigrationFile.js
 packages/lib/services/database/migrations/42.js
 packages/lib/services/database/migrations/43.js
 packages/lib/services/database/migrations/44.js
+packages/lib/services/database/migrations/45.js
+packages/lib/services/database/sqlStringToLines.js
 packages/lib/services/database/types.js
 packages/lib/services/debug/populateDatabase.js
 packages/lib/services/e2ee/EncryptionService.test.js
@@ -799,6 +801,13 @@ packages/lib/services/keychain/KeychainServiceDriverBase.js
 packages/lib/services/noteList/defaultLeftToRightListRenderer.js
 packages/lib/services/noteList/defaultListRenderer.js
 packages/lib/services/noteList/renderers.js
+packages/lib/services/ocr/OcrDriverBase.js
+packages/lib/services/ocr/OcrService.test.js
+packages/lib/services/ocr/OcrService.js
+packages/lib/services/ocr/drivers/OcrDriverTesseract.js
+packages/lib/services/ocr/utils/filterOcrText.test.js
+packages/lib/services/ocr/utils/filterOcrText.js
+packages/lib/services/ocr/utils/types.js
 packages/lib/services/plugins/BasePlatformImplementation.js
 packages/lib/services/plugins/BasePluginRunner.js
 packages/lib/services/plugins/MenuController.js
@@ -876,6 +885,7 @@ packages/lib/services/rest/utils/paginatedResults.js
 packages/lib/services/rest/utils/readonlyProperties.js
 packages/lib/services/rest/utils/requestFields.js
 packages/lib/services/rest/utils/requestPaginationOptions.js
+packages/lib/services/searchengine/SearchEngine.resources.test.js
 packages/lib/services/searchengine/SearchEngine.js
 packages/lib/services/searchengine/SearchEngineUtils.test.js
 packages/lib/services/searchengine/SearchEngineUtils.js
@@ -923,6 +933,7 @@ packages/lib/services/synchronizer/utils/handleSyncStartupOperation.js
 packages/lib/services/synchronizer/utils/resourceRemotePath.js
 packages/lib/services/synchronizer/utils/syncDeleteStep.js
 packages/lib/services/synchronizer/utils/types.js
+packages/lib/shim-init-node.js
 packages/lib/shim.js
 packages/lib/testing/syncTargetUtils.js
 packages/lib/testing/test-utils-synchronizer.js

.github/workflows/build-macos-m1.yml

@@ -33,6 +33,12 @@ jobs:
           # https://yarnpkg.com/getting-started/install
           corepack enable
 
+      - name: Install macOs dependencies
+        if: runner.os == 'macOS'
+        run: |
+          # Required for building the canvas package
+          brew install pango
+
       # See github-action-main.yml for explanation
       - uses: actions/setup-python@v4
         with:

.github/workflows/github-actions-main.yml

@@ -59,6 +59,12 @@ jobs:
           # testing.
           sudo apt-get install -y xvfb
 
+      - name: Install macOs dependencies
+        if: runner.os == 'macOS'
+        run: |
+          # Required for building the canvas package
+          brew install pango
+
       - name: Install Docker Engine
         # if: runner.os == 'Linux' && startsWith(github.ref, 'refs/tags/server-v')
         if: runner.os == 'Linux'

.gitignore

@@ -733,6 +733,8 @@ packages/lib/services/database/addMigrationFile.js
 packages/lib/services/database/migrations/42.js
 packages/lib/services/database/migrations/43.js
 packages/lib/services/database/migrations/44.js
+packages/lib/services/database/migrations/45.js
+packages/lib/services/database/sqlStringToLines.js
 packages/lib/services/database/types.js
 packages/lib/services/debug/populateDatabase.js
 packages/lib/services/e2ee/EncryptionService.test.js
@@ -779,6 +781,13 @@ packages/lib/services/keychain/KeychainServiceDriverBase.js
 packages/lib/services/noteList/defaultLeftToRightListRenderer.js
 packages/lib/services/noteList/defaultListRenderer.js
 packages/lib/services/noteList/renderers.js
+packages/lib/services/ocr/OcrDriverBase.js
+packages/lib/services/ocr/OcrService.test.js
+packages/lib/services/ocr/OcrService.js
+packages/lib/services/ocr/drivers/OcrDriverTesseract.js
+packages/lib/services/ocr/utils/filterOcrText.test.js
+packages/lib/services/ocr/utils/filterOcrText.js
+packages/lib/services/ocr/utils/types.js
 packages/lib/services/plugins/BasePlatformImplementation.js
 packages/lib/services/plugins/BasePluginRunner.js
 packages/lib/services/plugins/MenuController.js
@@ -856,6 +865,7 @@ packages/lib/services/rest/utils/paginatedResults.js
 packages/lib/services/rest/utils/readonlyProperties.js
 packages/lib/services/rest/utils/requestFields.js
 packages/lib/services/rest/utils/requestPaginationOptions.js
+packages/lib/services/searchengine/SearchEngine.resources.test.js
 packages/lib/services/searchengine/SearchEngine.js
 packages/lib/services/searchengine/SearchEngineUtils.test.js
 packages/lib/services/searchengine/SearchEngineUtils.js
@@ -903,6 +913,7 @@ packages/lib/services/synchronizer/utils/handleSyncStartupOperation.js
 packages/lib/services/synchronizer/utils/resourceRemotePath.js
 packages/lib/services/synchronizer/utils/syncDeleteStep.js
 packages/lib/services/synchronizer/utils/types.js
+packages/lib/shim-init-node.js
 packages/lib/shim.js
 packages/lib/testing/syncTargetUtils.js
 packages/lib/testing/test-utils-synchronizer.js

.yarn/patches/pdfjs-dist-npm-3.11.174-67f2fee6d6.patch

@@ -0,0 +1,15 @@
+
+# We remove the `canvas` optional dependency because electron-rebuild fails to build it, and
+# the `canvas` API is already part of Electron
+diff --git a/package.json b/package.json
+index 105811f53d508486e08a60dc1b6e437cd24d7427..dea6a4e6612c4a4006cc482e46ff5270dcfda1e5 100644
+--- a/package.json
++++ b/package.json
+@@ -13,7 +13,6 @@
+   "bugs": "https://github.com/mozilla/pdf.js/issues",
+   "license": "Apache-2.0",
+   "optionalDependencies": {
+-    "canvas": "^2.11.2",
+     "path2d-polyfill": "^2.0.1"
+   },
+   "browser": {

package.json

@@ -31,7 +31,7 @@
     "crowdinUpload": "crowdin upload",
     "cspell": "cspell",
     "dependencyTree": "madge",
-    "generateDatabaseTypes": "node packages/tools/generate-database-types",
+    "generateTypes": "node packages/tools/generate-database-types",
     "linkChecker": "linkchecker https://joplinapp.org/ && linkchecker --check-extern https://joplinapp.org/api/references/plugin_api/classes/joplin.html",
     "linter-ci": "eslint --resolve-plugins-relative-to . --quiet --ext .js --ext .jsx --ext .ts --ext .tsx",
     "linter-interactive": "eslint-interactive --resolve-plugins-relative-to . --fix --quiet --ext .js --ext .jsx --ext .ts --ext .tsx",
@@ -105,6 +105,7 @@
     "react-native-vosk@0.1.12": "patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch",
     "eslint": "patch:eslint@8.52.0#./.yarn/patches/eslint-npm-8.39.0-d92bace04d.patch",
     "app-builder-lib@24.4.0": "patch:app-builder-lib@npm%3A24.4.0#./.yarn/patches/app-builder-lib-npm-24.4.0-05322ff057.patch",
-    "react-native@0.71.10": "patch:react-native@npm%3A0.71.10#./.yarn/patches/react-native-animation-fix/react-native-npm-0.71.10-f9c32562d8.patch"
+    "react-native@0.71.10": "patch:react-native@npm%3A0.71.10#./.yarn/patches/react-native-animation-fix/react-native-npm-0.71.10-f9c32562d8.patch",
+    "pdfjs-dist": "patch:pdfjs-dist@npm%3A3.11.174#./.yarn/patches/pdfjs-dist-npm-3.11.174-67f2fee6d6.patch"
   }
 }

packages/app-desktop/.gitignore

@@ -14,6 +14,8 @@ style.min.css
 build/lib/
 vendor/*
 !vendor/loadEmojiLib.js
+build/pdf.worker.min.js
+build/tesseract.js*
 test-results/
 playwright-report/
 playwright/.cache/

packages/app-desktop/app.ts

@@ -63,11 +63,14 @@ import ShareService from '@joplin/lib/services/share/ShareService';
 import checkForUpdates from './checkForUpdates';
 import { AppState } from './app.reducer';
 import syncDebugLog from '@joplin/lib/services/synchronizer/syncDebugLog';
-import eventManager from '@joplin/lib/eventManager';
+import eventManager, { EventName } from '@joplin/lib/eventManager';
 import path = require('path');
 import { checkPreInstalledDefaultPlugins, installDefaultPlugins, setSettingsForDefaultPlugins } from '@joplin/lib/services/plugins/defaultPlugins/defaultPluginsUtils';
 import userFetcher, { initializeUserFetcher } from '@joplin/lib/utils/userFetcher';
 import { parseNotesParent } from '@joplin/lib/reducer';
+import OcrService from '@joplin/lib/services/ocr/OcrService';
+import OcrDriverTesseract from '@joplin/lib/services/ocr/drivers/OcrDriverTesseract';
+import SearchEngine from '@joplin/lib/services/searchengine/SearchEngine';
 import { PackageInfo } from '@joplin/lib/versionInfo';
 
 const pluginClasses = [
@@ -83,6 +86,7 @@ class Application extends BaseApplication {
 
 	private checkAllPluginStartedIID_: any = null;
 	private initPluginServiceDone_ = false;
+	private ocrService_: OcrService;
 
 	public constructor() {
 		super();
@@ -121,6 +125,10 @@ class Application extends BaseApplication {
 			this.updateTray();
 		}
 
+		if (action.type === 'SETTING_UPDATE_ONE' && action.key === 'ocr.enabled' || action.type === 'SETTING_UPDATE_ALL') {
+			this.setupOcrService();
+		}
+
 		if (action.type === 'SETTING_UPDATE_ONE' && action.key === 'style.editor.fontFamily' || action.type === 'SETTING_UPDATE_ALL') {
 			this.updateEditorFont();
 		}
@@ -355,6 +363,34 @@ class Application extends BaseApplication {
 		Setting.setValue('wasClosedSuccessfully', false);
 	}
 
+	private setupOcrService() {
+		if (Setting.value('ocr.enabled')) {
+			if (!this.ocrService_) {
+				const Tesseract = (window as any).Tesseract;
+
+				const driver = new OcrDriverTesseract(
+					{ createWorker: Tesseract.createWorker },
+					`${bridge().buildDir()}/tesseract.js/worker.min.js`,
+					`${bridge().buildDir()}/tesseract.js-core`,
+				);
+
+				this.ocrService_ = new OcrService(driver);
+			}
+
+			void this.ocrService_.runInBackground();
+		} else {
+			if (!this.ocrService_) return;
+			void this.ocrService_.stopRunInBackground();
+		}
+
+		const handleResourceChange = () => {
+			void this.ocrService_.maintenance();
+		};
+
+		eventManager.on(EventName.ResourceCreate, handleResourceChange);
+		eventManager.on(EventName.ResourceChange, handleResourceChange);
+	}
+
 	public async start(argv: string[]): Promise<any> {
 		// If running inside a package, the command line, instead of being "node.exe <path> <flags>" is "joplin.exe <flags>" so
 		// insert an extra argument so that they can be processed in a consistent way everywhere.
@@ -571,7 +607,7 @@ class Application extends BaseApplication {
 		// Forwards the local event to the global event manager, so that it can
 		// be picked up by the plugin manager.
 		ResourceEditWatcher.instance().on('resourceChange', (event: any) => {
-			eventManager.emit('resourceChange', event);
+			eventManager.emit(EventName.ResourceChange, event);
 		});
 
 		RevisionService.instance().runInBackground();
@@ -587,6 +623,8 @@ class Application extends BaseApplication {
 				bridge: bridge(),
 				debug: new DebugService(reg.db()),
 				resourceService: ResourceService.instance(),
+				searchEngine: SearchEngine.instance(),
+				ocrService: () => this.ocrService_,
 			};
 		}
 
@@ -600,6 +638,12 @@ class Application extends BaseApplication {
 
 		this.startRotatingLogMaintenance(Setting.value('profileDir'));
 
+		await this.setupOcrService();
+
+		eventManager.on(EventName.OcrServiceResourcesProcessed, () => {
+			SearchEngine.instance().scheduleSyncTables();
+		});
+
 		// await populateDatabase(reg.db(), {
 		// 	clearDatabase: true,
 		// 	folderCount: 1000,

packages/app-desktop/gui/MainScreen/commands/editAlarm.ts

@@ -1,5 +1,5 @@
 import { CommandRuntime, CommandDeclaration, CommandContext } from '@joplin/lib/services/CommandService';
-import eventManager from '@joplin/lib/eventManager';
+import eventManager, { EventName } from '@joplin/lib/eventManager';
 import { _ } from '@joplin/lib/locale';
 import { stateUtils } from '@joplin/lib/reducer';
 import Note from '@joplin/lib/models/Note';
@@ -45,7 +45,7 @@ export const runtime = (comp: any): CommandRuntime => {
 
 						if (newNote) {
 							await Note.save(newNote);
-							eventManager.emit('alarmChange', { noteId: note.id, note: newNote });
+							eventManager.emit(EventName.AlarmChange, { noteId: note.id, note: newNote });
 						}
 
 						comp.setState({ promptOptions: null });

packages/app-desktop/gui/MainScreen/commands/toggleNoteType.ts

@@ -1,7 +1,7 @@
 import { CommandRuntime, CommandDeclaration, CommandContext } from '@joplin/lib/services/CommandService';
 import { _ } from '@joplin/lib/locale';
 import Note from '@joplin/lib/models/Note';
-import eventManager from '@joplin/lib/eventManager';
+import eventManager, { EventName } from '@joplin/lib/eventManager';
 
 export const declaration: CommandDeclaration = {
 	name: 'toggleNoteType',
@@ -22,7 +22,7 @@ export const runtime = (): CommandRuntime => {
 					todo_due: newNote.todo_due,
 					todo_completed: newNote.todo_completed,
 				};
-				eventManager.emit('noteTypeToggle', { noteId: note.id, note: eventNote });
+				eventManager.emit(EventName.NoteTypeToggle, { noteId: note.id, note: eventNote });
 			}
 		},
 		enabledCondition: '!noteIsReadOnly',

packages/app-desktop/gui/MenuBar.tsx

@@ -25,6 +25,7 @@ import { ProfileConfig } from '@joplin/lib/services/profileConfig/types';
 import PluginService, { PluginSettings } from '@joplin/lib/services/plugins/PluginService';
 import { getListRendererById, getListRendererIds } from '@joplin/lib/services/noteList/renderers';
 import useAsyncEffect from '@joplin/lib/hooks/useAsyncEffect';
+import { EventName } from '@joplin/lib/eventManager';
 const packageInfo: PackageInfo = require('../packageInfo.js');
 const { clipboard } = require('electron');
 const Menu = bridge().Menu;
@@ -1011,10 +1012,10 @@ function useMenu(props: Props) {
 			setKeymapLastChangeTime(Date.now());
 		}
 
-		KeymapService.instance().on('keymapChange', onKeymapChange);
+		KeymapService.instance().on(EventName.KeymapChange, onKeymapChange);
 
 		return () => {
-			KeymapService.instance().off('keymapChange', onKeymapChange);
+			KeymapService.instance().off(EventName.KeymapChange, onKeymapChange);
 		};
 	}, []);
 

packages/app-desktop/gui/NoteEditor/NoteBody/CodeMirror/utils/useKeymap.ts

@@ -5,6 +5,7 @@ import { EditorCommand } from '../../../utils/types';
 import shim from '@joplin/lib/shim';
 import { reg } from '@joplin/lib/registry';
 import setupVim from './setupVim';
+import { EventName } from '@joplin/lib/eventManager';
 
 export default function useKeymap(CodeMirror: any) {
 
@@ -174,7 +175,7 @@ export default function useKeymap(CodeMirror: any) {
 		const keymapService = KeymapService.instance();
 
 		registerKeymap();
-		keymapService.on('keymapChange', registerKeymap);
+		keymapService.on(EventName.KeymapChange, registerKeymap);
 
 		setupEmacs();
 		setupVim(CodeMirror);