reactions: Store the emoji codepoint in the database.

This is the first part of a larger migration to convert Zulip's
reactions storage to something based on the codepoint, not the emoji
name that the user typed in, so that we don't need to worry about
changes in the names we're using breaking the emoji storage.

static/js/reactions.js

@@ -128,7 +128,9 @@ exports.add_reaction = function (event) {
         return;
     }
 
-    event.emoji_name_css_class = emoji.emojis_name_to_css_class[emoji_name];
+    if (event.reaction_type === 'unicode_emoji') {
+        event.emoji_name_css_class = event.emoji_code;
+    }
     event.user.id = event.user.user_id;
 
     message.reactions.push(event);
@@ -267,25 +269,23 @@ exports.get_message_reactions = function (message) {
                           'in reaction for message ' + message.id);
             return;
         }
-
-        var user_list = message_reactions.setdefault(reaction.emoji_name, []);
-        user_list.push(user_id);
+        reaction.user_ids = [];
+        var collapsed_reaction = message_reactions.setdefault(reaction.emoji_name, reaction);
+        collapsed_reaction.user_ids.push(user_id);
     });
     var reactions = message_reactions.items().map(function (item) {
         var emoji_name = item[0];
-        var user_ids = item[1];
-        var reaction = {
-            emoji_name: emoji_name,
-            emoji_name_css_class: emoji.emojis_name_to_css_class[emoji_name],
-            count: user_ids.length,
-            title: generate_title(emoji_name, user_ids),
-            emoji_alt_code: page_params.emoji_alt_code,
-        };
-        if (emoji.realm_emojis[reaction.emoji_name]) {
+        var reaction = item[1];
+        reaction.emoji_name_css_class = reaction.emoji_code;
+        reaction.count = reaction.user_ids.length;
+        reaction.title = generate_title(reaction.emoji_name, reaction.user_ids);
+        reaction.emoji_alt_code = page_params.emoji_alt_code;
+
+        if (reaction.reaction_type !== 'unicode_emoji') {
             reaction.is_realm_emoji = true;
             reaction.url = emoji.realm_emojis[reaction.emoji_name].emoji_url;
         }
-        if (user_ids.indexOf(page_params.user_id) !== -1) {
+        if (reaction.user_ids.indexOf(page_params.user_id) !== -1) {
             reaction.class = "message_reaction reacted";
         } else {
             reaction.class = "message_reaction";

zerver/lib/actions.py

@@ -20,6 +20,7 @@
     to_dict_cache_key_id,
 )
 from zerver.lib.context_managers import lockfile
+from zerver.lib.emoji import emoji_name_to_emoji_code
 from zerver.lib.hotspots import get_next_hotspots
 from zerver.lib.message import (
     access_message,
@@ -939,8 +940,8 @@ def do_send_messages(messages_maybe_none):
     # intermingle sending zephyr messages with other messages.
     return already_sent_ids + [message['message'].id for message in messages]
 
-def notify_reaction_update(user_profile, message, emoji_name, op):
-    # type: (UserProfile, Message, Text, Text) -> None
+def notify_reaction_update(user_profile, message, reaction, op):
+    # type: (UserProfile, Message, Reaction, Text) -> None
     user_dict = {'user_id': user_profile.id,
                  'email': user_profile.email,
                  'full_name': user_profile.full_name}
@@ -949,7 +950,9 @@ def notify_reaction_update(user_profile, message, emoji_name, op):
              'op': op,
              'user': user_dict,
              'message_id': message.id,
-             'emoji_name': emoji_name}  # type: Dict[str, Any]
+             'emoji_name': reaction.emoji_name,
+             'emoji_code': reaction.emoji_code,
+             'reaction_type': reaction.reaction_type}  # type: Dict[str, Any]
 
     # Update the cached message since new reaction is added.
     update_to_dict_cache([message])
@@ -970,16 +973,20 @@ def notify_reaction_update(user_profile, message, emoji_name, op):
 
 def do_add_reaction(user_profile, message, emoji_name):
     # type: (UserProfile, Message, Text) -> None
-    reaction = Reaction(user_profile=user_profile, message=message, emoji_name=emoji_name)
+    (emoji_code, reaction_type) = emoji_name_to_emoji_code(user_profile.realm, emoji_name)
+    reaction = Reaction(user_profile=user_profile, message=message,
+                        emoji_name=emoji_name, emoji_code=emoji_code,
+                        reaction_type=reaction_type)
     reaction.save()
-    notify_reaction_update(user_profile, message, emoji_name, "add")
+    notify_reaction_update(user_profile, message, reaction, "add")
 
 def do_remove_reaction(user_profile, message, emoji_name):
     # type: (UserProfile, Message, Text) -> None
-    Reaction.objects.filter(user_profile=user_profile,
-                            message=message,
-                            emoji_name=emoji_name).delete()
-    notify_reaction_update(user_profile, message, emoji_name, "remove")
+    reaction = Reaction.objects.filter(user_profile=user_profile,
+                                       message=message,
+                                       emoji_name=emoji_name).get()
+    reaction.delete()
+    notify_reaction_update(user_profile, message, reaction, "remove")
 
 def do_send_typing_notification(notification):
     # type: (Dict[str, Any]) -> None

zerver/lib/emoji.py

@@ -2,25 +2,38 @@
 
 import os
 import re
+import ujson
 
 from django.utils.translation import ugettext as _
-from typing import Optional, Text
-from zerver.lib.bugdown import name_to_codepoint
+from typing import Optional, Text, Tuple
+
 from zerver.lib.request import JsonableError
 from zerver.lib.upload import upload_backend
-from zerver.models import Realm, RealmEmoji, UserProfile
+from zerver.models import Reaction, Realm, RealmEmoji, UserProfile
 
-def check_valid_emoji(realm, emoji_name):
-    # type: (Realm, Text) -> None
+# Until migration to iamcal dataset is complete use the unified
+# reactions file to convert a reaction emoji name to codepoint.
+# Once the migration is complete this will be switched to use
+# name_to_codepoint map.
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+UNIFIED_REACTIONS_FILE_PATH = os.path.join(ZULIP_PATH, 'zerver', 'management', 'data', 'unified_reactions.json')
+unified_reactions = ujson.load(open(UNIFIED_REACTIONS_FILE_PATH))
+
+def emoji_name_to_emoji_code(realm, emoji_name):
+    # type: (Realm, Text) -> Tuple[Text, Text]
     realm_emojis = realm.get_emoji()
     if emoji_name in realm_emojis and not realm_emojis[emoji_name]['deactivated']:
-        return
-    if emoji_name in name_to_codepoint:
-        return
+        return emoji_name, Reaction.REALM_EMOJI
     if emoji_name == 'zulip':
-        return
+        return emoji_name, Reaction.FAKE_REALM_EMOJI
+    if emoji_name in unified_reactions:
+        return unified_reactions[emoji_name], Reaction.UNICODE_EMOJI
     raise JsonableError(_("Emoji '%s' does not exist" % (emoji_name,)))
 
+def check_valid_emoji(realm, emoji_name):
+    # type: (Realm, Text) -> None
+    emoji_name_to_emoji_code(realm, emoji_name)
+
 def check_emoji_admin(user_profile, emoji_name=None):
     # type: (UserProfile, Optional[Text]) -> None
     """Raises an exception if the user cannot administer the target realm

zerver/lib/message.py

@@ -247,6 +247,8 @@ class ReactionDict(object):
     def build_dict_from_raw_db_row(row):
         # type: (Dict[str, Any]) -> Dict[str, Any]
         return {'emoji_name': row['emoji_name'],
+                'emoji_code': row['emoji_code'],
+                'reaction_type': row['reaction_type'],
                 'user': {'email': row['user_profile__email'],
                          'id': row['user_profile__id'],
                          'full_name': row['user_profile__full_name']}}

zerver/migrations/0085_reactions_emoji_code.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.2 on 2017-06-18 21:26
+from __future__ import unicode_literals
+
+import os
+import ujson
+
+from django.conf import settings
+from django.db import migrations, models
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+def populate_new_fields(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
+    # Open the JSON file which contains the data to be used for migration.
+    MIGRATION_DATA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "management", "data")
+    path_to_unified_reactions = os.path.join(MIGRATION_DATA_PATH, "unified_reactions.json")
+    unified_reactions = ujson.load(open(path_to_unified_reactions))
+
+    Reaction = apps.get_model('zerver', 'Reaction')
+    for reaction in Reaction.objects.all():
+        reaction.emoji_code = unified_reactions.get(reaction.emoji_name)
+        if reaction.emoji_code is None:
+            # If it's not present in the unified_reactions map, it's a realm emoji.
+            reaction.codepoint = reaction.emoji_name
+            if reaction.emoji_name == 'zulip':
+                # `:zulip:` emoji is a fake realm emoji.
+                reaction.reaction_type = 'fake_realm_emoji'
+            else:
+                reaction.reaction_type = 'realm_emoji'
+        reaction.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zerver', '0084_realmemoji_deactivated'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='reaction',
+            name='emoji_code',
+            field=models.TextField(default='unset'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='reaction',
+            name='reaction_type',
+            field=models.CharField(choices=[('unicode_emoji', 'Unicode emoji'), ('realm_emoji', 'Realm emoji'), ('fake_realm_emoji', 'Fake realm emoji')], default='unicode_emoji', max_length=30),
+        ),
+        migrations.RunPython(populate_new_fields,
+                             reverse_code=migrations.RunPython.noop),
+    ]

zerver/models.py

@@ -1232,15 +1232,25 @@ class Reaction(ModelReprMixin, models.Model):
     user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE) # type: UserProfile
     message = models.ForeignKey(Message, on_delete=CASCADE) # type: Message
     emoji_name = models.TextField() # type: Text
+    emoji_code = models.TextField() # type: Text
+
+    UNICODE_EMOJI       = u'unicode_emoji'
+    REALM_EMOJI         = u'realm_emoji'
+    FAKE_REALM_EMOJI    = u'fake_realm_emoji'
+    REACTION_TYPES         = ((UNICODE_EMOJI, _("Unicode emoji")),
+                              (REALM_EMOJI, _("Realm emoji")),
+                              (FAKE_REALM_EMOJI, _("Fake realm emoji")))
+
+    reaction_type = models.CharField(default=UNICODE_EMOJI, choices=REACTION_TYPES, max_length=30) # type: Text
 
     class Meta(object):
         unique_together = ("user_profile", "message", "emoji_name")
 
     @staticmethod
     def get_raw_db_rows(needed_ids):
         # type: (List[int]) -> List[Dict[str, Any]]
-        fields = ['message_id', 'emoji_name', 'user_profile__email',
-                  'user_profile__id', 'user_profile__full_name']
+        fields = ['message_id', 'emoji_name', 'emoji_code', 'reaction_type',
+                  'user_profile__email', 'user_profile__id', 'user_profile__full_name']
         return Reaction.objects.filter(message_id__in=needed_ids).values(*fields)
 
 # Whenever a message is sent, for each user current subscribed to the