Desktop: Add support for OneNote importer

.eslintignore

@@ -83,6 +83,7 @@ plugin_types/
 readme/
 packages/react-native-vosk/lib/
 packages/lib/countable/Countable.js
+packages/onenote-converter/pkg/onenote_converter.js
 
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
 packages/app-cli/app/LinkSelector.js
@@ -1002,6 +1003,8 @@ packages/lib/services/interop/InteropService_Importer_Md.test.js
 packages/lib/services/interop/InteropService_Importer_Md.js
 packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.js
 packages/lib/services/interop/InteropService_Importer_Md_frontmatter.js
+packages/lib/services/interop/InteropService_Importer_OneNote.test.js
+packages/lib/services/interop/InteropService_Importer_OneNote.js
 packages/lib/services/interop/InteropService_Importer_Raw.test.js
 packages/lib/services/interop/InteropService_Importer_Raw.js
 packages/lib/services/interop/Module.test.js

.gitignore

@@ -982,6 +982,8 @@ packages/lib/services/interop/InteropService_Importer_Md.test.js
 packages/lib/services/interop/InteropService_Importer_Md.js
 packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.js
 packages/lib/services/interop/InteropService_Importer_Md_frontmatter.js
+packages/lib/services/interop/InteropService_Importer_OneNote.test.js
+packages/lib/services/interop/InteropService_Importer_OneNote.js
 packages/lib/services/interop/InteropService_Importer_Raw.test.js
 packages/lib/services/interop/InteropService_Importer_Raw.js
 packages/lib/services/interop/Module.test.js

Dockerfile.server

@@ -33,6 +33,7 @@ COPY packages/renderer ./packages/renderer
 COPY packages/tools ./packages/tools
 COPY packages/utils ./packages/utils
 COPY packages/lib ./packages/lib
+COPY packages/onenote-converter ./packages/onenote-converter
 COPY packages/server ./packages/server
 
 # For some reason there's both a .yarn/cache and .yarn/berry/cache that are

joplin.code-workspace

@@ -5,6 +5,9 @@
 		},
 	],
 	"settings": {
+		"rust-analyzer.linkedProjects": [
+			"./packages/onenote-converter/Cargo.toml",
+		],
 		"files.exclude": {
 			"_mydocs/mdtest/": true,
 			"_releases/": true,

package.json

@@ -2,6 +2,7 @@
   "name": "root",
   "private": true,
   "workspaces": [
+    "packages/onenote-converter/pkg",
     "packages/*"
   ],
   "repository": {

packages/lib/BaseModel.ts

@@ -1,11 +1,11 @@
 import paginationToSql from './models/utils/paginationToSql';
 import Database from './database';
-import uuid from './uuid';
 import time from './time';
 import JoplinDatabase, { TableField } from './JoplinDatabase';
 import { LoadOptions, SaveOptions } from './models/utils/types';
 import ActionLogger, { ItemActionType as ItemActionType } from './utils/ActionLogger';
 import { SqlQuery } from './services/database/types';
+import uuid from './uuid';
 const Mutex = require('async-mutex').Mutex;
 
 // New code should make use of this enum
@@ -80,6 +80,8 @@ class BaseModel {
 		['TYPE_COMMAND', ModelType.Command],
 	];
 
+	private static uuidGenerator: ()=> string = uuid.create;
+
 	public static TYPE_NOTE = ModelType.Note;
 	public static TYPE_FOLDER = ModelType.Folder;
 	public static TYPE_SETTING = ModelType.Setting;
@@ -573,7 +575,7 @@ class BaseModel {
 
 		if (options.isNew) {
 			if (this.useUuid() && !o.id) {
-				modelId = uuid.create();
+				modelId = this.generateUuid();
 				o.id = modelId;
 			}
 
@@ -754,6 +756,13 @@ class BaseModel {
 		return this.db_;
 	}
 
+	public static generateUuid() {
+		return this.uuidGenerator();
+	}
+
+	public static setIdGenerator(generator: ()=> string) {
+		this.uuidGenerator = generator;
+	}
 	// static isReady() {
 	// 	return !!this.db_;
 	// }

packages/lib/htmlUtils.ts

@@ -172,7 +172,6 @@ class HtmlUtils {
 
 		return output.join(' ');
 	}
-
 }
 
 export default new HtmlUtils();

packages/lib/package.json

@@ -17,6 +17,7 @@
   },
   "devDependencies": {
     "@testing-library/react-hooks": "8.0.1",
+    "@types/adm-zip": "0.5.5",
     "@types/fs-extra": "11.0.4",
     "@types/jest": "29.5.8",
     "@types/js-yaml": "4.0.9",
@@ -44,11 +45,13 @@
     "@joplin/fork-sax": "^1.2.55",
     "@joplin/fork-uslug": "^1.0.16",
     "@joplin/htmlpack": "~3.0",
+    "@joplin/onenote-converter": "0.0.1",
     "@joplin/renderer": "~3.0",
     "@joplin/turndown": "^4.0.73",
     "@joplin/turndown-plugin-gfm": "^1.0.55",
     "@joplin/utils": "~3.0",
     "@types/nanoid": "3.0.0",
+    "adm-zip": "0.5.12",
     "async-mutex": "0.4.1",
     "base-64": "1.0.0",
     "base64-stream": "1.0.0",

packages/lib/services/interop/InteropService.ts

@@ -19,6 +19,7 @@ import InteropService_Exporter_Md_frontmatter from './InteropService_Exporter_Md
 import InteropService_Importer_Base from './InteropService_Importer_Base';
 import InteropService_Exporter_Base from './InteropService_Exporter_Base';
 import Module, { dynamicRequireModuleFactory, makeExportModule, makeImportModule } from './Module';
+import InteropService_Importer_OneNote from './InteropService_Importer_OneNote';
 const { sprintf } = require('sprintf-js');
 const { fileExtension } = require('../../path-utils');
 const EventEmitter = require('events');
@@ -133,6 +134,14 @@ export default class InteropService {
 					isNoteArchive: false, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
 					description: _('Text document'),
 				}, () => new InteropService_Importer_Md()),
+
+				makeImportModule({
+					format: 'zip',
+					fileExtensions: ['zip'],
+					sources: [FileSystemItem.File],
+					isNoteArchive: false, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
+					description: _('OneNote Notebook'),
+				}, () => new InteropService_Importer_OneNote()),
 			];
 
 			const exportModules = [

packages/lib/services/interop/InteropService_Importer_Md.ts

@@ -12,7 +12,7 @@ import htmlUtils from '../../htmlUtils';
 import { unique } from '../../ArrayUtils';
 const { pregQuote } = require('../../string-utils-common');
 import { MarkupToHtml } from '@joplin/renderer';
-import { isDataUrl } from '@joplin/utils/url';
+import { isDataUrl, isMailTo, isFilenameTooLong } from '@joplin/utils/url';
 import { stripBom } from '../../string-utils';
 
 export default class InteropService_Importer_Md extends InteropService_Importer_Base {
@@ -108,11 +108,12 @@ export default class InteropService_Importer_Md extends InteropService_Importer_
 		let updated = md;
 		const markdownLinks = markdownUtils.extractFileUrls(md);
 		const htmlLinks = htmlUtils.extractFileUrls(md);
-		const fileLinks = unique(markdownLinks.concat(htmlLinks));
+		const pdfLinks = htmlUtils.extractPdfUrls(md);
+		const fileLinks = unique(markdownLinks.concat(htmlLinks).concat(pdfLinks));
 		for (const encodedLink of fileLinks) {
 			const link = decodeURI(encodedLink);
 
-			if (isDataUrl(link)) {
+			if (isDataUrl(link) || isMailTo(link) || isFilenameTooLong(link)) {
 				// Just leave it as it is. We could potentially import
 				// it as a resource but for now that's good enough.
 			} else {

packages/lib/services/interop/InteropService_Importer_OneNote.test.ts

@@ -0,0 +1,129 @@
+import Note from '../../models/Note';
+import Folder from '../../models/Folder';
+import * as fs from 'fs-extra';
+import { createTempDir, setupDatabaseAndSynchronizer, supportDir, switchClient } from '../../testing/test-utils';
+import { NoteEntity } from '../database/types';
+import InteropService_Importer_OneNote from './InteropService_Importer_OneNote';
+import { MarkupToHtml } from '@joplin/renderer';
+import BaseModel from '../../BaseModel';
+import uuid from '../../uuid';
+
+describe('InteropService_Importer_OneNote', () => {
+	let tempDir: string;
+	async function importNote(path: string) {
+		const newFolder = await Folder.save({ title: 'folder' });
+		const importer = new InteropService_Importer_OneNote();
+		await importer.init(path, {
+			format: 'md',
+			outputFormat: 'md',
+			path,
+			destinationFolder: newFolder,
+			destinationFolderId: newFolder.id,
+		});
+		importer.setMetadata({ fileExtensions: ['md'] });
+		await importer.exec({ warnings: [] });
+		const allNotes: NoteEntity[] = await Note.all();
+		return allNotes;
+	}
+	beforeEach(async () => {
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+		tempDir = await createTempDir();
+	});
+	afterEach(async () => {
+		await fs.remove(tempDir);
+	});
+	it('should import a simple OneNote notebook', async () => {
+		const notes = await importNote(`${supportDir}/onenote/simple_notebook.zip`);
+		const folders = await Folder.all();
+
+		expect(notes.length).toBe(2);
+		const mainNote = notes[0];
+
+		expect(folders.length).toBe(3);
+		const parentFolder = folders.find(f => f.id === mainNote.parent_id);
+		expect(parentFolder.title).toBe('Section title');
+		expect(folders.find(f => f.id === parentFolder.parent_id).title).toBe('Simple notebook');
+
+		expect(mainNote.title).toBe('Page title');
+		expect(mainNote.markup_language).toBe(MarkupToHtml.MARKUP_LANGUAGE_HTML);
+		expect(mainNote.body).toMatchSnapshot(mainNote.title);
+	});
+
+	it('should preserve indentation of subpages in Section page', async () => {
+		const notes = await importNote(`${supportDir}/onenote/subpages.zip`);
+
+		const sectionPage = notes.find(n => n.title === 'Section');
+		const menuHtml = sectionPage.body.split('<ul>')[1].split('</ul>')[0];
+		const menuLines = menuHtml.split('</li>');
+
+		const pageTwo = notes.find(n => n.title === 'Page 2');
+		expect(menuLines[3].trim()).toBe(`<li class="l1"><a href=":/${pageTwo.id}" target="content" title="Page 2">${pageTwo.title}</a>`);
+
+		const pageTwoA = notes.find(n => n.title === 'Page 2-a');
+		expect(menuLines[4].trim()).toBe(`<li class="l2"><a href=":/${pageTwoA.id}" target="content" title="Page 2-a">${pageTwoA.title}</a>`);
+
+		const pageTwoAA = notes.find(n => n.title === 'Page 2-a-a');
+		expect(menuLines[5].trim()).toBe(`<li class="l3"><a href=":/${pageTwoAA.id}" target="content" title="Page 2-a-a">${pageTwoAA.title}</a>`);
+
+		const pageTwoB = notes.find(n => n.title === 'Page 2-b');
+		expect(menuLines[7].trim()).toBe(`<li class="l2"><a href=":/${pageTwoB.id}" target="content" title="Page 2-b">${pageTwoB.title}</a>`);
+	});
+
+	it('should created subsections', async () => {
+		const notes = await importNote(`${supportDir}/onenote/subsections.zip`);
+		const folders = await Folder.all();
+
+		const parentSection = folders.find(f => f.title === 'Group Section 1');
+		const subSection = folders.find(f => f.title === 'Group Section 1-a');
+		const subSection1 = folders.find(f => f.title === 'Subsection 1');
+		const subSection2 = folders.find(f => f.title === 'Subsection 2');
+		const notesFromParentSection = notes.filter(n => n.parent_id === parentSection.id);
+
+		expect(parentSection.id).toBe(subSection1.parent_id);
+		expect(parentSection.id).toBe(subSection2.parent_id);
+		expect(parentSection.id).toBe(subSection.parent_id);
+		expect(folders.length).toBe(7);
+		expect(notes.length).toBe(6);
+		expect(notesFromParentSection.length).toBe(2);
+	});
+
+	it('should expect notes to be rendered the same', async () => {
+		let idx = 0;
+		BaseModel.setIdGenerator(() => String(idx++));
+		const notes = await importNote(`${supportDir}/onenote/complex_notes.zip`);
+
+		for (const note of notes) {
+			expect(note.body).toMatchSnapshot(note.title);
+		}
+		BaseModel.setIdGenerator(uuid.create);
+	});
+
+	it('should render the proper tree for notebook with group sections', async () => {
+		const notes = await importNote(`${supportDir}/onenote/group_sections.zip`);
+		const folders = await Folder.all();
+
+		const mainFolder = folders.find(f => f.title === 'Notebook created on OneNote App');
+		const section = folders.find(f => f.title === 'Section');
+		const sectionA1 = folders.find(f => f.title === 'Section A1');
+		const sectionA = folders.find(f => f.title === 'Section A');
+		const sectionB1 = folders.find(f => f.title === 'Section B1');
+		const sectionB = folders.find(f => f.title === 'Section B');
+		const sectionD1 = folders.find(f => f.title === 'Section D1');
+		const sectionD = folders.find(f => f.title === 'Section D');
+
+		expect(section.parent_id).toBe(mainFolder.id);
+		expect(sectionA.parent_id).toBe(mainFolder.id);
+		expect(sectionD.parent_id).toBe(mainFolder.id);
+
+		expect(sectionA1.parent_id).toBe(sectionA.id);
+		expect(sectionB.parent_id).toBe(sectionA.id);
+
+		expect(sectionB1.parent_id).toBe(sectionB.id);
+		expect(sectionD1.parent_id).toBe(sectionD.id);
+
+		expect(notes.filter(n => n.parent_id === sectionA1.id).length).toBe(2);
+		expect(notes.filter(n => n.parent_id === sectionB1.id).length).toBe(2);
+		expect(notes.filter(n => n.parent_id === sectionD1.id).length).toBe(1);
+	});
+});

packages/lib/services/interop/InteropService_Importer_OneNote.ts

@@ -0,0 +1,70 @@
+import { ImportExportResult, ImportModuleOutputFormat } from './types';
+
+import InteropService_Importer_Base from './InteropService_Importer_Base';
+import { NoteEntity } from '../database/types';
+import { rtrimSlashes } from '../../path-utils';
+import { oneNoteConverter } from '@joplin/onenote-converter';
+import * as AdmZip from 'adm-zip';
+import InteropService_Importer_Md from './InteropService_Importer_Md';
+import { join, resolve } from 'path';
+import Logger from '@joplin/utils/Logger';
+import path = require('path');
+
+const logger = Logger.create('InteropService_Importer_OneNote');
+
+export default class InteropService_Importer_OneNote extends InteropService_Importer_Base {
+	protected importedNotes: Record<string, NoteEntity> = {};
+
+	private getEntryDirectory(unzippedPath: string, entryName: string) {
+		const withoutBasePath = entryName.replace(unzippedPath, '');
+		return path.normalize(withoutBasePath).split(path.sep)[0];
+	}
+
+	public async exec(result: ImportExportResult) {
+		const sourcePath = rtrimSlashes(this.sourcePath_);
+		const unzipTempDirectory = await this.temporaryDirectory_(true);
+		const zip = new AdmZip(sourcePath);
+		logger.info('Unzipping files...');
+		zip.extractAllTo(unzipTempDirectory, false);
+
+		const files = zip.getEntries();
+		if (files.length === 0) {
+			result.warnings.push('Zip file has no files.');
+			return result;
+		}
+
+		// files that don't have a name seems to be local only and shouldn't be processed
+
+		const tempOutputDirectory = await this.temporaryDirectory_(true);
+		const baseFolder = this.getEntryDirectory(unzipTempDirectory, files[0].entryName);
+		const notebookBaseDir = path.join(unzipTempDirectory, baseFolder, path.sep);
+		const outputDirectory2 = path.join(tempOutputDirectory, baseFolder);
+
+		const notebookFiles = zip.getEntries().filter(e => e.name !== '.onetoc2' && e.name !== 'OneNote_RecycleBin.onetoc2');
+
+		logger.info('Extracting OneNote to HTML');
+		for (const notebookFile of notebookFiles) {
+			const notebookFilePath = join(unzipTempDirectory, notebookFile.entryName);
+			try {
+				await oneNoteConverter(notebookFilePath, resolve(outputDirectory2), notebookBaseDir);
+			} catch (error) {
+				console.error(error);
+			}
+		}
+
+		logger.info('Importing HTML into Joplin');
+		const importer = new InteropService_Importer_Md();
+		importer.setMetadata({ fileExtensions: ['html'] });
+		await importer.init(tempOutputDirectory, {
+			...this.options_,
+			format: 'html',
+			outputFormat: ImportModuleOutputFormat.Html,
+
+		});
+		logger.info('Finished');
+		result = await importer.exec(result);
+
+		// remover temp directories?
+		return result;
+	}
+}