Implement markdown parser to convert markdown to html

.eslintrc

@@ -111,6 +111,7 @@
         "Entypo", "msup", "mrow", "webview", "js", "timerow", "reselect", "addons", "cancelable",
         "gravatar_hash", "pms", "msgs",
         "collapsable", "const", "wildcard",
+        "camelcase"
       ],
       "skipIfMatch": [
         "http://[^s]*",

package.json

@@ -23,7 +23,8 @@
     "test:e2e": "detox build; jest -t '__e2e__tests__/basic.js'",
     "test:coveralls": "jest --coverage && cat ./coverage/lcov.info | coveralls",
     "test:prettier": "prettier-check  --single-quote --jsx-bracket-same-line --trailing-comma all --print-width 100 --parser flow `find src -name \"*.js\"`",
-    "test": "npm run test:lint && npm run test:prettier && npm run test:coveralls && npm run test:flow"
+    "test": "npm run test:lint && npm run test:prettier && npm run test:coveralls && npm run test:flow",
+    "test-temp": "jest preview"
   },
   "dependencies": {
     "@expo/react-native-action-sheet": "0.3.1",
@@ -37,7 +38,12 @@
     "events": "^1.1.1",
     "htmlparser2": "^3.9.2",
     "immutable": "^3.8.1",
+    "lodash.contains": "^2.4.3",
+    "lodash.find": "^4.6.0",
+    "lodash.foreach": "^4.5.0",
     "lodash.isequal": "^4.4.0",
+    "lodash.map": "^4.6.0",
+    "lodash.reject": "^4.6.0",
     "lodash.throttle": "^4.1.1",
     "lodash.uniqby": "^4.4.0",
     "react": "16.0.0-alpha.12",

src/chat/preview/__tests__/preview-tests.js

@@ -0,0 +1,223 @@
+/* eslint-disable camelcase*/
+
+import parseMarkdown from '../parseMarkdown';
+
+const testCases = [
+  { input: 'hello', expected: '<p>hello</p>' },
+  { input: 'hello there', expected: '<p>hello there</p>' },
+  { input: 'hello **bold** for you', expected: '<p>hello <strong>bold</strong> for you</p>' },
+  { input: '__hello__', expected: '<p>__hello__</p>' },
+  {
+    input: '\n```\nfenced code\n```\n\nand then after\n',
+    expected:
+      '<div class="codehilite"><pre><span></span>fenced code\n</pre></div>\n\n\n<p>and then after</p>',
+  },
+  {
+    input: '\n```\n    fenced code trailing whitespace            \n```\n\nand then after\n',
+    expected:
+      '<div class="codehilite"><pre><span></span>    fenced code trailing whitespace\n</pre></div>\n\n\n<p>and then after</p>',
+  },
+  {
+    input: '* a\n* list \n* here',
+    expected: '<ul>\n<li>a</li>\n<li>list </li>\n<li>here</li>\n</ul>',
+  },
+  {
+    input: '\n```c#\nfenced code special\n```\n\nand then after\n',
+    expected:
+      '<div class="codehilite"><pre><span></span>fenced code special\n</pre></div>\n\n\n<p>and then after</p>',
+  },
+  {
+    input: '\n```vb.net\nfenced code dot\n```\n\nand then after\n',
+    expected:
+      '<div class="codehilite"><pre><span></span>fenced code dot\n</pre></div>\n\n\n<p>and then after</p>',
+  },
+  {
+    input: 'Some text first\n* a\n* list \n* here\n\nand then after',
+    expected:
+      '<p>Some text first</p>\n<ul>\n<li>a</li>\n<li>list </li>\n<li>here</li>\n</ul>\n<p>and then after</p>',
+  },
+  {
+    input: '1. an\n2. ordered \n3. list',
+    expected: '<p>1. an<br>\n2. ordered<br>\n3. list</p>',
+  },
+  {
+    input: '\n~~~quote\nquote this for me\n~~~\nthanks\n',
+    expected: '<blockquote>\n<p>quote this for me</p>\n</blockquote>\n<p>thanks</p>',
+  },
+  {
+    input: 'This is a @**Cordelia Lear** mention',
+    expected:
+      '<p>This is a <span class="user-mention" data-user-id="101">@Cordelia Lear</span> mention</p>',
+  },
+  {
+    input: 'These @ @**** are not mentions',
+    expected: '<p>These @ @<em>**</em> are not mentions</p>',
+  },
+  {
+    input: 'These # #**** are not mentions',
+    expected: '<p>These # #<em>**</em> are not mentions</p>',
+  },
+  {
+    input: 'These @* @*** are not mentions',
+    expected: '<p>These @* @*** are not mentions</p>',
+  },
+  {
+    input: 'These #* #*** are also not mentions',
+    expected: '<p>These #* #*** are also not mentions</p>',
+  },
+  {
+    input: 'This is a #**Denmark** stream link',
+    expected:
+      '<p>This is a <a class="stream" data-stream-id="1" href="http://localhost:9991/#narrow/stream/Denmark">#Denmark</a> stream link</p>',
+  },
+  {
+    input: 'This is #**Denmark** and #**social** stream links',
+    expected:
+      '<p>This is <a class="stream" data-stream-id="1" href="http://localhost:9991/#narrow/stream/Denmark">#Denmark</a> and <a class="stream" data-stream-id="2" href="http://localhost:9991/#narrow/stream/social">#social</a> stream links</p>',
+  },
+  {
+    input: 'And this is a #**wrong** stream link',
+    expected: '<p>And this is a #**wrong** stream link</p>',
+  },
+  {
+    input: 'mmm...:burrito:s',
+    expected:
+      '<p>mmm...<img alt=":burrito:" class="emoji" src="/static/generated/emoji/images/emoji/unicode/1f32f.png" title="burrito">s</p>',
+  },
+  {
+    input: 'This is an :poop: message',
+    expected:
+      '<p>This is an <img alt=":poop:" class="emoji" src="/static/generated/emoji/images/emoji/unicode/1f4a9.png" title="poop"> message</p>',
+  },
+  {
+    input: '\ud83d\udca9',
+    expected:
+      '<p><img alt=":poop:" class="emoji" src="/static/generated/emoji/images/emoji/unicode/1f4a9.png" title="poop"></p>',
+  },
+  {
+    input: '\u{1f937}',
+    expected: '<p>\u{1f937}</p>',
+  },
+  {
+    input: 'This is a realm filter #1234 with text after it',
+    expected:
+      '<p>This is a realm filter <a href="https://trac.zulip.net/ticket/1234" target="_blank" title="https://trac.zulip.net/ticket/1234">#1234</a> with text after it</p>',
+  },
+  {
+    input: 'This is a realm filter with ZGROUP_123:45 groups',
+    expected:
+      '<p>This is a realm filter with <a href="https://zone_45.zulip.net/ticket/123" target="_blank" title="https://zone_45.zulip.net/ticket/123">ZGROUP_123:45</a> groups</p>',
+  },
+  {
+    input: 'This is an !avatar(cordelia@zulip.com) of Cordelia Lear',
+    expected:
+      '<p>This is an <img alt="cordelia@zulip.com" class="message_body_gravatar" src="/avatar/cordelia@zulip.com?s=30" title="cordelia@zulip.com"> of Cordelia Lear</p>',
+  },
+  {
+    input: 'This is a !gravatar(cordelia@zulip.com) of Cordelia Lear',
+    expected:
+      '<p>This is a <img alt="cordelia@zulip.com" class="message_body_gravatar" src="/avatar/cordelia@zulip.com?s=30" title="cordelia@zulip.com"> of Cordelia Lear</p>',
+  },
+  {
+    input: 'Test *italic*',
+    expected: '<p>Test <em>italic</em></p>',
+  },
+  {
+    input: 'T\n#**Denmark**',
+    expected:
+      '<p>T<br>\n<a class="stream" data-stream-id="1" href="http://localhost:9991/#narrow/stream/Denmark">#Denmark</a></p>',
+  },
+  {
+    input: 'T\n@**Cordelia Lear**',
+    expected: '<p>T<br>\n<span class="user-mention" data-user-id="101">@Cordelia Lear</span></p>',
+  },
+  {
+    input: 'This is a realm filter `hello` with text after it',
+    expected: '<p>This is a realm filter <code>hello</code> with text after it</p>',
+  },
+];
+
+const users = [
+  {
+    fullName: 'Cordelia Lear',
+    id: 101,
+    email: 'cordelia@zulip.com',
+  },
+  {
+    fullName: 'Leo',
+    id: 102,
+    email: 'leo@zulip.com',
+  },
+  {
+    fullName: 'Iago',
+    id: 103,
+    email: 'iago@zulip.com',
+  },
+];
+const streams = [
+  {
+    subscribed: false,
+    color: 'blue',
+    name: 'Denmark',
+    stream_id: 1,
+    in_home_view: false,
+  },
+  {
+    subscribed: true,
+    color: 'red',
+    name: 'social',
+    stream_id: 2,
+    in_home_view: true,
+    invite_only: true,
+  },
+];
+const auth = {
+  realm: 'http://localhost:9991',
+  apiKey: 'AJHDFIAS8231827381',
+  email: 'iago@zulip.com',
+};
+const realm_users = [];
+const realm_emoji = {
+  burrito: {
+    display_url: '/static/generated/emoji/images/emoji/burrito.png',
+    source_url: '/static/generated/emoji/images/emoji/burrito.png',
+  },
+};
+const realm_filters = [
+  ['#(?P<id>[0-9]{2,8})', 'https://trac.zulip.net/ticket/%(id)s'],
+  ['ZBUG_(?P<id>[0-9]{2,8})', 'https://trac2.zulip.net/ticket/%(id)s'],
+  [
+    'ZGROUP_(?P<id>[0-9]{2,8}):(?P<zone>[0-9]{1,8})',
+    'https://zone_%(zone)s.zulip.net/ticket/%(id)s',
+  ],
+];
+describe('Preview test messages', () => {
+  // const testCase = testCases[20];
+  // test(`Test markdown ${testCase.input.replace('\n', '\\n')}`, () => {
+  //   const parsedHTML = parseMarkdown(
+  //     testCase.input,
+  //     users,
+  //     streams,
+  //     auth,
+  //     realm_users,
+  //     realm_filters,
+  //     realm_emoji,
+  //   );
+  //   expect(parsedHTML).toEqual(testCase.expected);
+  // });
+  let index = 0;
+  testCases.forEach(testCase =>
+    test(`Test index ${index++} markdown ${testCase.input.replace('\n', '\\n')}`, () => {
+      const parsedHTML = parseMarkdown(
+        testCase.input,
+        users,
+        streams,
+        auth,
+        realm_users,
+        realm_filters,
+        realm_emoji,
+      );
+      expect(parsedHTML).toEqual(testCase.expected);
+    }),
+  );
+});

src/chat/preview/emoji.js

@@ -0,0 +1,125 @@
+// prettier-disable
+/* eslint-disable */
+// prettier-ignore
+var forEach = require('lodash.foreach');
+var emoji_codes = require('./emoji_codes');
+var emoji = (function (realm_emoji) {
+
+var exports = {};
+
+exports.emojis = [];
+exports.all_realm_emojis = {};
+exports.active_realm_emojis = {};
+exports.emojis_by_name = {};
+exports.emojis_name_to_css_class = {};
+exports.emojis_by_unicode = {};
+
+var default_emojis = [];
+var default_unicode_emojis = [];
+
+
+var zulip_emoji = {
+    emoji_name: 'zulip',
+    emoji_url: '/static/generated/emoji/images/emoji/unicode/zulip.png',
+    is_realm_emoji: true,
+    deactivated: false,
+};
+
+forEach(emoji_codes.names, function (value) {
+    var base_name = emoji_codes.name_to_codepoint[value];
+    default_emojis.push({emoji_name: value,
+                         codepoint: emoji_codes.name_to_codepoint[value],
+                         emoji_url: "/static/generated/emoji/images/emoji/unicode/" + base_name + ".png"});
+});
+
+forEach(emoji_codes.codepoints, function (value) {
+    default_unicode_emojis.push({emoji_name: value,
+                                 codepoint: value,
+                                 emoji_url: "/static/generated/emoji/images/emoji/unicode/" + value + ".png"});
+});
+
+exports.update_emojis = function update_emojis(realm_emojis) {
+    // exports.all_realm_emojis is emptied before adding the realm-specific emoji to it.
+    // This makes sure that in case of deletion, the deleted realm_emojis don't
+    // persist in exports.all_realm_emojis or exports.active_realm_emojis.
+    exports.all_realm_emojis = {};
+    exports.active_realm_emojis = {};
+
+    // Copy the default emoji list and add realm-specific emoji to it
+    exports.emojis = default_emojis.slice(0);
+    forEach(realm_emojis, function (data, name) {
+        exports.all_realm_emojis[name] = {emoji_name: name,
+                                          emoji_url: data.source_url,
+                                          deactivated: data.deactivated};
+        if (data.deactivated !== true) {
+            // export.emojis are used in composebox autocomplete. This condition makes sure
+            // that deactivated emojis don't appear in the autocomplete.
+            exports.emojis.push({emoji_name: name,
+                                 emoji_url: data.source_url,
+                                 is_realm_emoji: true});
+            exports.active_realm_emojis[name] = {emoji_name: name, emoji_url: data.source_url};
+        }
+    });
+    // Add the Zulip emoji to the realm emojis list
+    exports.emojis.push(zulip_emoji);
+    exports.all_realm_emojis.zulip = zulip_emoji;
+    exports.active_realm_emojis.zulip = zulip_emoji;
+
+    exports.emojis_by_name = {};
+    exports.emojis_name_to_css_class = {};
+    forEach(default_emojis, function (emoji) {
+        var css_class = emoji_codes.name_to_codepoint[emoji.emoji_name];
+        exports.emojis_name_to_css_class[emoji.emoji_name] = css_class;
+        exports.emojis_by_name[emoji.emoji_name] = emoji.emoji_url;
+    });
+    // Code for patching CSS classes for flag emojis so that they render
+    // properly in emoji picker. Remove after migration to iamcal dataset
+    // is complete.
+    forEach(emoji_codes.patched_css_classes, function (css_class, name) {
+        exports.emojis_name_to_css_class[name] = css_class;
+    });
+    exports.emojis_by_unicode = {};
+    forEach(default_unicode_emojis, function (emoji) {
+        exports.emojis_by_unicode[emoji.emoji_name] = emoji.emoji_url;
+    });
+};
+
+exports.initialize = function initialize() {
+    // Load the sprite image in the background so that the browser
+    // can cache it for later use.
+    var sprite = new Image();
+    sprite.src = '/static/generated/emoji/sheet_google_32.png';
+};
+
+exports.set_realm_emoji = function(realm_emoji) {
+  realm_emoji = realm_emoji;
+};
+
+exports.update_emojis(realm_emoji);
+
+
+exports.build_emoji_upload_widget = function () {
+
+    var get_file_input = function () {
+        return $('#emoji_file_input');
+    };
+
+    var file_name_field = $('#emoji-file-name');
+    var input_error = $('#emoji_file_input_error');
+    var clear_button = $('#emoji_image_clear_button');
+    var upload_button = $('#emoji_upload_button');
+
+    return upload_widget.build_widget(
+        get_file_input,
+        file_name_field,
+        input_error,
+        clear_button,
+        upload_button
+    );
+};
+
+return exports;
+}());
+if (typeof module !== 'undefined') {
+    module.exports = emoji;
+}