feat: replace weird spaces characters

adobe · Apr 5, 2022 · 4c9b43f · 4c9b43f
1 parent cbbadfb
commit 4c9b43f
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 4 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -7,7 +7,7 @@
         {
             "type": "node",
             "request": "launch",
-            "name": "Mocha Current File",
+            "name": "Mocha All",
             "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",
             "args": [
                 "--timeout",

diff --git a/package-lock.json b/package-lock.json
diff --git a/src/importer/PageImporter.js b/src/importer/PageImporter.js
@@ -231,7 +231,7 @@ export default class PageImporter {
   }
 
   postProcessMD(md) {
-    return md.replace(/\\\\~/gm, '\\~');
+    return MDUtils.cleanupMarkdown(md);
   }
 
   async download(url) {

diff --git a/src/utils/MDUtils.js b/src/utils/MDUtils.js
@@ -18,4 +18,20 @@ export default class MDUtils {
       return md.replace(new RegExp(`${oldSrc.replace('.', '\\.').replace('?', '\\?')}`, 'gm'), newSrc);
     }
   };
+
+  static cleanupMarkdown(md) {
+    let ret = md?.replace(/\\\\~/gm, '\\~');
+    if (ret) {
+      for (let i = 0; i < 20; i += 1) {
+        let x = `${i}`;
+        if (i < 10) x = `0${i}`;
+        const c = String.fromCodePoint(parseInt(`00${x}`, 16));
+        const reg = new RegExp(`\\u00${x}`, 'g');
+        const r = [c.length].map(() => ' ').join('');
+        ret = ret.replace(reg, r);
+      }
+      ret = ret.replace(/\u00A0/gm, '');
+    }
+    return ret;
+  }
 }
diff --git a/test/utils/MDUtils.spec.js b/test/utils/MDUtils.spec.js
@@ -15,7 +15,7 @@ import { describe, it } from 'mocha';
 
 import MDUtils from '../../src/utils/MDUtils.js';
 
-describe('MDUtils tests', () => {
+describe('MDUtils#replaceSrcInMarkdown tests', () => {
   it('MDUtils#replaceSrcInMarkdown do nothing', () => {
     strictEqual(
       MDUtils.replaceSrcInMarkdown('#A title\n![](https://www.sample.com/image.jpg)', 'https://www.sample.com/replace.jpg', 'https://www.sample.com/withimage.jpg'),
@@ -52,3 +52,30 @@ describe('MDUtils tests', () => {
     );
   });
 });
+
+describe('MDUtils#cleanupMarkdown tests', () => {
+  it('MDUtils#cleanupMarkdown do nothing', () => {
+    const md = '#A title\n![](https://www.sample.com/image.jpg)\nShould be clean\n<table><tr><th colspan="2">Metadata</th></tr><tr><td>key</td><td>value</td></tr></table>';
+    strictEqual(
+      MDUtils.cleanupMarkdown(md),
+      md,
+      'do nothing',
+    );
+  });
+
+  it('MDUtils#cleanupMarkdown unescape tildes', () => {
+    strictEqual(
+      MDUtils.cleanupMarkdown('#A title\n~~Tilde can be an pb~~\n Especially \\\\~in the content'),
+      '#A title\n~~Tilde can be an pb~~\n Especially \\~in the content',
+      'unescape tildes',
+    );
+  });
+
+  it('MDUtils#cleanupMarkdown replace weird spaces', () => {
+    strictEqual(
+      MDUtils.cleanupMarkdown('#A title\nReplaces the weird spaces characters: "\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019"'),
+      '#A title\nReplaces the weird spaces characters: "                    "',
+      'replace weird spaces',
+    );
+  });
+});