Merge branch 'dev' into release-2.6

packages/app-cli/tests/enex_to_html/quoted-attributes.enex

@@ -0,0 +1,3 @@
+<en-note>
+  <h1 style="box-sizing:inherit;font-family:&quot;Guardian TextSans Web&quot;, &quot;Helvetica Neue&quot;, Helvetica, Arial, sans-serif;margin-top:0.2em;margin-bottom:0.35em;font-size:2.125em;font-weight:600;line-height:1.3;">Association Between mRNA Vaccination and COVID-19 Hospitalization and Disease Severity</h1>
+</en-note>

packages/app-cli/tests/enex_to_html/quoted-attributes.html

@@ -0,0 +1,3 @@
+<en-note>
+  <h1 style="box-sizing:inherit;font-family:&quot;Guardian TextSans Web&quot;, &quot;Helvetica Neue&quot;, Helvetica, Arial, sans-serif;margin-top:0.2em;margin-bottom:0.35em;font-size:2.125em;font-weight:600;line-height:1.3;">Association Between mRNA Vaccination and COVID-19 Hospitalization and Disease Severity</h1>
+</en-note>

packages/app-cli/tests/enex_to_md/highlight.html

@@ -1 +1,3 @@
-<span style="background-color: rgb(255, 250, 165);-evernote-highlight:true;">I&apos;ll highlight some text.</span>
+<span style="background-color: rgb(255, 250, 165);-evernote-highlight:true;">I&apos;ll highlight some text.</span>
+<br/>
+<span style="--en-highlight:yellow;background-color: #ffef9e;">this text is yellow</span>

packages/app-cli/tests/enex_to_md/highlight.md

@@ -1 +1,2 @@
-==I'll highlight some text.==
+==I'll highlight some text.==
+==this text is yellow==

packages/app-cli/tests/html_to_html/anchor_missing_href.dest.html

@@ -0,0 +1 @@
+<a data-from-md href='#'>test</a>

packages/app-cli/tests/html_to_html/anchor_missing_href.src.html

@@ -0,0 +1 @@
+<a>test</a>

packages/lib/import-enex-html-gen.test.js

@@ -0,0 +1,122 @@
+
+const { setupDatabaseAndSynchronizer, switchClient, supportDir } = require('./testing/test-utils.js');
+const shim = require('./shim').default;
+const { enexXmlToHtml } = require('./import-enex-html-gen.js');
+const cleanHtml = require('clean-html');
+
+const fileWithPath = (filename) =>
+	`${supportDir}/../enex_to_html/${filename}`;
+
+const audioResource = {
+	filename: 'audio test',
+	id: '9168ee833d03c5ea7c730ac6673978c1',
+	mime: 'audio/x-m4a',
+	size: 82011,
+	title: 'audio test',
+};
+
+// All the test HTML files are beautified ones, so we need to run
+// this before the comparison. Before, beautifying was done by `enexXmlToHtml`
+// but that was removed due to problems with the clean-html package.
+const beautifyHtml = (html) => {
+	return new Promise((resolve) => {
+		try {
+			cleanHtml.clean(html, { wrap: 0 }, (...cleanedHtml) => resolve(cleanedHtml.join('')));
+		} catch (error) {
+			console.warn(`Could not clean HTML - the "unclean" version will be used: ${error.message}: ${html.trim().substr(0, 512).replace(/[\n\r]/g, ' ')}...`);
+			resolve([html].join(''));
+		}
+	});
+};
+
+/**
+ * Tests the importer for a single note, checking that the result of
+ * processing the given `.enex` input file matches the contents of the given
+ * `.html` file.
+ *
+ * Note that this does not test the importing of an entire exported `.enex`
+ * archive, but rather a single node of such a file. Thus, the test data files
+ * (e.g. `./enex_to_html/code1.enex`) correspond to the contents of a single
+ * `<note>...</note>` node in an `.enex` file already extracted from
+ * `<content><![CDATA[...]]</content>`.
+ */
+const compareOutputToExpected = (options) => {
+	options = {
+		resources: [],
+		...options,
+	};
+
+	const inputFile = fileWithPath(`${options.testName}.enex`);
+	const outputFile = fileWithPath(`${options.testName}.html`);
+	const testTitle = `should convert from Enex to Html: ${options.testName}`;
+
+	it(testTitle, (async () => {
+		const enexInput = await shim.fsDriver().readFile(inputFile);
+		const expectedOutput = await shim.fsDriver().readFile(outputFile);
+		const actualOutput = await beautifyHtml(await enexXmlToHtml(enexInput, options.resources));
+		expect(actualOutput).toEqual(expectedOutput);
+	}));
+};
+
+describe('EnexToHtml', function() {
+	beforeEach(async (done) => {
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+		done();
+	});
+
+	compareOutputToExpected({
+		testName: 'checklist-list',
+	});
+
+	compareOutputToExpected({
+		testName: 'svg',
+	});
+
+	compareOutputToExpected({
+		testName: 'en-media--image',
+		resources: [{
+			filename: '',
+			id: '89ce7da62c6b2832929a6964237e98e9', // Mock id
+			mime: 'image/jpeg',
+			size: 50347,
+			title: '',
+		}],
+	});
+
+	compareOutputToExpected({
+		testName: 'en-media--audio',
+		resources: [audioResource],
+	});
+
+	compareOutputToExpected({
+		testName: 'attachment',
+		resources: [{
+			filename: 'attachment-1',
+			id: '21ca2b948f222a38802940ec7e2e5de3',
+			mime: 'application/pdf', // Any non-image/non-audio mime type will do
+			size: 1000,
+		}],
+	});
+
+	compareOutputToExpected({
+		testName: 'quoted-attributes',
+	});
+
+	// it('fails when not given a matching resource', (async () => {
+	// 	// To test the promise-unexpectedly-resolved case, add `audioResource` to the array.
+	// 	const resources = [];
+	// 	const inputFile = fileWithPath('en-media--image.enex');
+	// 	const enexInput = await shim.fsDriver().readFile(inputFile);
+	// 	const promisedOutput = enexXmlToHtml(enexInput, resources);
+
+	// 	promisedOutput.then(() => {
+	// 		// Promise should not be resolved
+	// 		expect(false).toEqual(true);
+	// 	}, (reason) => {
+	// 		expect(reason)
+	// 			.toBe('Hash with no associated resource: 89ce7da62c6b2832929a6964237e98e9');
+	// 	});
+	// }));
+
+});

packages/lib/import-enex-md-gen.ts

@@ -426,14 +426,21 @@ function attributeToLowerCase(node: any) {
 	return output;
 }
 
-function cssValue(context: any, style: string, propName: string): string {
+function cssValue(context: any, style: string, propName: string | string[]): string {
 	if (!style) return null;
 
+	const propNames = Array.isArray(propName) ? propName : [propName];
+
 	try {
 		const o = cssParser.parse(`pre {${style}}`);
 		if (!o.stylesheet.rules.length) return null;
-		const prop = o.stylesheet.rules[0].declarations.find((d: any) => d.property.toLowerCase() === propName);
-		return prop && prop.value ? prop.value.trim().toLowerCase() : null;
+
+		for (const propName of propNames) {
+			const prop = o.stylesheet.rules[0].declarations.find((d: any) => d.property.toLowerCase() === propName);
+			if (prop && prop.value) return prop.value.trim().toLowerCase();
+		}
+
+		return null;
 	} catch (error) {
 		displaySaxWarning(context, error.message);
 		return null;
@@ -507,7 +514,13 @@ function isCodeBlock(context: any, nodeName: string, attributes: any) {
 		// Yes, this property sometimes appears as -en-codeblock, sometimes as
 		// --en-codeblock. Would be too easy to import ENEX data otherwise.
 		// https://github.com/laurent22/joplin/issues/4965
-		const enCodeBlock = cssValue(context, attributes.style, '-en-codeblock') || cssValue(context, attributes.style, '--en-codeblock');
+		const enCodeBlock = cssValue(context, attributes.style, [
+			'-en-codeblock',
+			'--en-codeblock',
+			'-evernote-codeblock',
+			'--evernote-codeblock',
+		]);
+
 		if (enCodeBlock && enCodeBlock.toLowerCase() === 'true') return true;
 	}
 	return false;
@@ -518,8 +531,19 @@ function isHighlight(context: any, _nodeName: string, attributes: any) {
 		// Evernote uses various inconsistent CSS prefixes: so far I've found
 		// "--en", "-en", "-evernote", so I'm guessing "--evernote" probably
 		// exists too.
-		const enHighlight = cssValue(context, attributes.style, '-evernote-highlight') || cssValue(context, attributes.style, '--evernote-highlight');
-		if (enHighlight && enHighlight.toLowerCase() === 'true') return true;
+
+		const enHighlight = cssValue(context, attributes.style, [
+			'-evernote-highlight',
+			'--evernote-highlight',
+			'-en-highlight',
+			'--en-highlight',
+		]);
+
+		// Value can be any colour or "true". I guess if it's set at all it
+		// should be highlighted but just in case handle case where it's
+		// "false".
+
+		if (enHighlight && enHighlight.toLowerCase() !== 'false') return true;
 	}
 	return false;
 }

packages/lib/resourceUtils.js

@@ -39,7 +39,7 @@ const imageMimeTypes = [
 	'image/vnd.xiff',
 ];
 
-const escapeQuotes = (str) => str.replace(/"/g, '"');
+const escapeQuotes = (str) => str.replace(/"/g, '"');
 
 const attributesToStr = (attributes) =>
 	Object.entries(attributes)

packages/renderer/htmlUtils.ts

@@ -192,6 +192,15 @@ class HtmlUtils {
 					}
 				}
 
+				// For some reason, entire parts of HTML notes don't show up in
+				// the viewer when there's an anchor tag without an "href"
+				// attribute. It doesn't always happen and it seems to depend on
+				// what else is in the note but in any case adding the "href"
+				// fixes it. https://github.com/laurent22/joplin/issues/5687
+				if (name.toLowerCase() === 'a' && !attrs['href']) {
+					attrs['href'] = '#';
+				}
+
 				let attrHtml = this.attributesHtml(attrs);
 				if (attrHtml) attrHtml = ` ${attrHtml}`;
 				const closingSign = this.isSelfClosingTag(name) ? '/>' : '>';

packages/server/src/app.ts

@@ -5,7 +5,7 @@ import * as Koa from 'koa';
 import * as fs from 'fs-extra';
 import Logger, { LoggerWrapper, TargetType } from '@joplin/lib/Logger';
 import config, { initConfig, runningInDocker } from './config';
-import { migrateLatest, waitForConnection, sqliteDefaultDir, latestMigration, DbConnection } from './db';
+import { migrateLatest, waitForConnection, sqliteDefaultDir, latestMigration } from './db';
 import { AppContext, Env, KoaNext } from './utils/types';
 import FsDriverNode from '@joplin/lib/fs-driver-node';
 import routeHandler from './middleware/routeHandler';
@@ -17,11 +17,10 @@ import startServices from './utils/startServices';
 import { credentialFile } from './utils/testing/testUtils';
 import apiVersionHandler from './middleware/apiVersionHandler';
 import clickJackingHandler from './middleware/clickJackingHandler';
-import newModelFactory, { Options } from './models/factory';
+import newModelFactory from './models/factory';
 import setupCommands from './utils/setupCommands';
 import { RouteResponseFormat, routeResponseFormat } from './utils/routeUtils';
 import { parseEnv } from './env';
-import storageDriverFromConfig from './models/items/storage/storageDriverFromConfig';
 
 interface Argv {
 	env?: Env;
@@ -222,13 +221,6 @@ async function main() {
 		fs.writeFileSync(pidFile, `${process.pid}`);
 	}
 
-	const newModelFactoryOptions = async (db: DbConnection): Promise<Options> => {
-		return {
-			storageDriver: await storageDriverFromConfig(config().storageDriver, db, { assignDriverId: env !== 'buildTypes' }),
-			storageDriverFallback: await storageDriverFromConfig(config().storageDriverFallback, db, { assignDriverId: env !== 'buildTypes' }),
-		};
-	};
-
 	let runCommandAndExitApp = true;
 
 	if (selectedCommand) {
@@ -245,7 +237,7 @@ async function main() {
 			});
 		} else {
 			const connectionCheck = await waitForConnection(config().database);
-			const models = newModelFactory(connectionCheck.connection, config(), await newModelFactoryOptions(connectionCheck.connection));
+			const models = newModelFactory(connectionCheck.connection, config());
 
 			await selectedCommand.run(commandArgv, {
 				db: connectionCheck.connection,
@@ -275,7 +267,7 @@ async function main() {
 		appLogger().info('Connection check:', connectionCheckLogInfo);
 		const ctx = app.context as AppContext;
 
-		await setupAppContext(ctx, env, connectionCheck.connection, appLogger, await newModelFactoryOptions(connectionCheck.connection));
+		await setupAppContext(ctx, env, connectionCheck.connection, appLogger);
 
 		await initializeJoplinUtils(config(), ctx.joplinBase.models, ctx.joplinBase.services.mustache);
 

packages/server/src/migrations/20211105183559_storage.ts

@@ -5,10 +5,16 @@ export async function up(db: DbConnection): Promise<any> {
 	await db.schema.createTable('storages', (table: Knex.CreateTableBuilder) => {
 		table.increments('id').unique().primary().notNullable();
 		table.text('connection_string').notNullable();
+		table.bigInteger('updated_time').notNullable();
+		table.bigInteger('created_time').notNullable();
 	});
 
+	const now = Date.now();
+
 	await db('storages').insert({
 		connection_string: 'Type=Database',
+		updated_time: now,
+		created_time: now,
 	});
 
 	// First we create the column and set a default so as to populate the
@@ -21,6 +27,10 @@ export async function up(db: DbConnection): Promise<any> {
 	await db.schema.alterTable('items', (table: Knex.CreateTableBuilder) => {
 		table.integer('content_storage_id').notNullable().alter();
 	});
+
+	await db.schema.alterTable('storages', (table: Knex.CreateTableBuilder) => {
+		table.unique(['connection_string']);
+	});
 }
 
 export async function down(db: DbConnection): Promise<any> {