Fixes errors caused by special characters (#17)

* Add `cleanString` function

* Update yarn.lock

web/app/components/document/sidebar.js

@@ -5,6 +5,7 @@ import { getOwner } from "@ember/application";
 import { inject as service } from "@ember/service";
 import { task } from "ember-concurrency";
 import { dasherize } from "@ember/string";
+import cleanString from "hermes/utils/clean-string";
 
 export default class DocumentSidebar extends Component {
   @service("fetch") fetchSvc;
@@ -212,13 +213,16 @@ export default class DocumentSidebar extends Component {
   *save(field, val) {
     if (field && val) {
       const oldVal = this[field];
-      this[field] = val;
+      this[field] = cleanString(val);
 
       try {
         const serializedValue = this.emailFields.includes(field)
           ? val.map((p) => p.email)
           : val;
-        yield this.patchDocument.perform({ [field]: serializedValue });
+
+        yield this.patchDocument.perform({
+          [field]: cleanString(serializedValue),
+        });
       } catch (err) {
         // revert field value on failure
         this[field] = oldVal;

web/app/components/new/doc-form.js

@@ -3,7 +3,8 @@ import { task } from "ember-concurrency";
 import { inject as service } from "@ember/service";
 import { tracked } from "@glimmer/tracking";
 import { action } from "@ember/object";
-import Ember from 'ember';
+import Ember from "ember";
+import cleanString from "hermes/utils/clean-string";
 
 const FORM_ERRORS = {
   title: null,
@@ -144,8 +145,8 @@ export default class NewDocForm extends Component {
             owner: this.authenticatedUser.info.email,
             product: this.productArea,
             productAbbreviation: this.productAbbreviation,
-            summary: this.summary,
-            title: this.title,
+            summary: cleanString(this.summary),
+            title: cleanString(this.title),
             tags: this.tags,
           }),
         })
@@ -160,12 +161,14 @@ export default class NewDocForm extends Component {
       yield wait(AWAIT_DOC_DELAY);
 
       // Set modal on a delay so it appears after transition.
-      this.modalAlerts.setActive.perform("docCreated", AWAIT_DOC_CREATED_MODAL_DELAY);
+      this.modalAlerts.setActive.perform(
+        "docCreated",
+        AWAIT_DOC_CREATED_MODAL_DELAY
+      );
 
       this.router.transitionTo("authenticated.document", doc.id, {
         queryParams: { draft: true },
       });
-
     } catch (err) {
       this.docIsBeingCreated = false;
       // TODO: Handle error by using a toast and showing the create form again with

web/app/utils/clean-string.ts

@@ -0,0 +1,74 @@
+import replaceSpecialCharacters from "replace-special-characters";
+
+export default function cleanString(string: string): string {
+  // ‘, ’, ‚
+  let newString = string.replace(/[\u2018\u2019\u201A]/g, "'");
+  // “, ”, „
+  newString = newString.replace(/[\u201C\u201D\u201E]/g, '"');
+  // …
+  newString = newString.replace(/\u2026/g, "...");
+  // –
+  newString = newString.replace(/\u2013/g, "-");
+  // —
+  newString = newString.replace(/\u2014/g, "--");
+  // •, ·
+  newString = newString.replace(/[\u2022\u00B7]/g, "*");
+  // «
+  newString = newString.replace(/\u00AB/g, "<<");
+  // »
+  newString = newString.replace(/\u00BB/g, ">>");
+  // ‹
+  newString = newString.replace(/\u2039/g, "<");
+  // ›
+  newString = newString.replace(/\u203A/g, ">");
+  // ¢
+  newString = newString.replace(/\u00A2/g, "c");
+  // ©
+  newString = newString.replace(/\u00A9/g, "(C)");
+  // ™
+  newString = newString.replace(/\u2122/g, "(TM)");
+  // ®
+  newString = newString.replace(/\u00AE/g, "(R)");
+  // °
+  newString = newString.replace(/\u00B0/g, "(deg)");
+  // ±
+  newString = newString.replace(/\u00B1/g, "+/-");
+  // ×
+  newString = newString.replace(/\u00D7/g, "*");
+  // ÷
+  newString = newString.replace(/\u00F7/g, "/");
+  // ¼
+  newString = newString.replace(/\u00BC/g, "1/4");
+  // ½
+  newString = newString.replace(/\u00BD/g, "1/2");
+  // ¾
+  newString = newString.replace(/\u00BE/g, "3/4");
+  // √
+  newString = newString.replace(/\u221A/g, "sqrt");
+  // ¹
+  newString = newString.replace(/\u00B9/g, "^1");
+  // ²
+  newString = newString.replace(/\u00B2/g, "^2");
+  // ³
+  newString = newString.replace(/\u00B3/g, "^3");
+  // ⁿ
+  newString = newString.replace(/\u207F/g, "^n");
+  // ¿
+  newString = newString.replace(/\u00BF/g, "?");
+  // ¡
+  newString = newString.replace(/\u00A1/g, "!");
+  // ≥
+  newString = newString.replace(/\u2265/g, ">=");
+  // ≤
+  newString = newString.replace(/\u2264/g, "<=");
+  // ≠
+  newString = newString.replace(/\u2260/g, "!=");
+
+  // replace special characters e.g., é with e
+  newString = replaceSpecialCharacters(newString);
+
+  // remove non-ascii characters
+  newString = newString.replace(/[^\x00-\x7F]/g, "");
+
+  return newString;
+}

web/package.json

@@ -128,6 +128,7 @@
     "instantsearch.js": "^4.40.2",
     "miragejs": "^0.1.47",
     "postcss-scss": "^4.0.3",
+    "replace-special-characters": "^1.2.7",
     "tailwindcss": "^3.0.23",
     "torii": "^0.10.1",
     "webpack": "^5.70.0"

web/tests/unit/utils/clean-string-test.ts

@@ -0,0 +1,40 @@
+import { module, test } from "qunit";
+import cleanString from "hermes/utils/clean-string";
+
+module("Unit | Utility | clean-string", function () {
+  test("it correctly cleans strings", function (assert) {
+    assert.equal(cleanString("\u2018\u2019\u201A"), "'''");
+    assert.equal(cleanString("\u201C\u201D\u201E"), '"""');
+    assert.equal(cleanString("\u2026"), "...");
+    assert.equal(cleanString("\u2013"), "-");
+    assert.equal(cleanString("\u2014"), "--");
+    assert.equal(cleanString("\u2022\u00B7"), "**");
+    assert.equal(cleanString("\u00AB"), "<<");
+    assert.equal(cleanString("\u00BB"), ">>");
+    assert.equal(cleanString("\u2039"), "<");
+    assert.equal(cleanString("\u203A"), ">");
+    assert.equal(cleanString("\u00A2"), "c");
+    assert.equal(cleanString("\u00A9"), "(C)");
+    assert.equal(cleanString("\u2122"), "(TM)");
+    assert.equal(cleanString("\u00AE"), "(R)");
+    assert.equal(cleanString("\u00B0"), "(deg)");
+    assert.equal(cleanString("\u00B1"), "+/-");
+    assert.equal(cleanString("\u00D7"), "*");
+    assert.equal(cleanString("\u00F7"), "/");
+    assert.equal(cleanString("\u00BC"), "1/4");
+    assert.equal(cleanString("\u00BD"), "1/2");
+    assert.equal(cleanString("\u00BE"), "3/4");
+    assert.equal(cleanString("\u221A"), "sqrt");
+    assert.equal(cleanString("\u00B9"), "^1");
+    assert.equal(cleanString("\u00B2"), "^2");
+    assert.equal(cleanString("\u00B3"), "^3");
+    assert.equal(cleanString("\u207F"), "^n");
+    assert.equal(cleanString("\u00BF"), "?");
+    assert.equal(cleanString("\u00A1"), "!");
+    assert.equal(cleanString("\u2265"), ">=");
+    assert.equal(cleanString("\u2264"), "<=");
+    assert.equal(cleanString("\u2260"), "!=");
+    assert.equal(cleanString("pøkémöñ"), "pokemon");
+    assert.equal(cleanString("∑π¬µ∫≈π†ºª¶§∞£"), "");
+  });
+});

web/yarn.lock

@@ -10838,6 +10838,7 @@ __metadata:
     prettier: ^2.5.1
     qunit: ^2.17.2
     qunit-dom: ^1.6.0
+    replace-special-characters: ^1.2.7
     tailwindcss: ^3.0.23
     torii: ^0.10.1
     typescript: latest
@@ -15022,6 +15023,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"replace-special-characters@npm:^1.2.7":
+  version: 1.2.7
+  resolution: "replace-special-characters@npm:1.2.7"
+  checksum: cc7db80e2f63e27255cf9b3ef89c82260bb89335634f53fe8bfcd1a10bfc7603c583214dc105c05b7a6f9b97a2789ace16222664d160662ef332a02074b4c9f7
+  languageName: node
+  linkType: hard
+
 "require-directory@npm:^2.1.1":
   version: 2.1.1
   resolution: "require-directory@npm:2.1.1"