Casefold when processing email addresses

res/matrix-org/migration_template.eml

@@ -11,10 +11,13 @@ Content-Type: multipart/alternative;
 Content-Type: text/plain; charset=UTF-8
 Content-Disposition: inline
 
-Hi,
+Hello,
 
-This is a notification to let you know that matrix id %(mxid)s was associated with
-a depreciated version of your email address and has been deleted.
+In the past, identity services did not take case into account when creating and storing
+mxids. This led to your email address, %(to)s being stored with two mxids. A recent update
+changed this behavior and streamlined the mxids associated with your email address.
+This is a notification to let you know that matrix id %(mxid)s has been deleted in accordance
+with this update.
 
 
 About Matrix:

sydent/db/migration.py

@@ -5,38 +5,45 @@
 
 from sydent.util import json_decoder
 from sydent.util.emailutils import sendEmail
-from sydent.util.hash import sha256_and_url_safe_base64
 
 
-def update_assosc(self, conn: sqlite3.Connection):
-    # from table local_threepid_associations, select all associations where the medium is 'email'
+def update_local_associations(self, conn: sqlite3.Connection):
+    """Update the DB table local_threepid_associations so that all stored
+    emails are casefolded, and any duplicate mxid's associated with the
+    given email are deleted.
+
+    :return: None
+    """
     cur = conn.cursor()
 
     res = cur.execute(
         "SELECT address, mxid FROM local_threepid_associations WHERE medium = 'email'"
         "ORDER BY ts DESC"
     )
 
+    # a dict that associates an email address with correspoinding mxids and lookup hashes
     associations: Dict[str, List[Tuple[str, str, str]]] = {}
 
     # iterate through selected associations, casefold email, rehash it, and add to
     # associations dict
-    for row in res.fetchall():
-        casefold_address = row[0].casefold()
+    for address, mxid in res.fetchall():
+        casefold_address = address.casefold()
 
-        # rehash the email since hash functions are case-sensitive
-        pepper_result = cur.execute("SELECT lookup_pepper from hashing_metadata")
-        pepper = pepper_result.fetchone()[0]
-        combo = "%s %s %s" % (casefold_address, "email", pepper)
-        lookup_hash = sha256_and_url_safe_base64(combo)
+        # rehash email since hashes are case-sensitive
+        lookup_hash = self.calculate_lookup(casefold_address)
 
         if casefold_address in associations:
-            associations[casefold_address].append((row[0], row[1], lookup_hash))
+            associations[casefold_address].append((address, mxid, lookup_hash))
         else:
-            associations[casefold_address] = [(row[0], row[1], lookup_hash)]
+            associations[casefold_address] = [(address, mxid, lookup_hash)]
 
+    # list of arguments to update db with
     db_update_args: List[Tuple[str, str, str, str]] = []
-    to_delete: List[str] = []
+
+    # list of mxids to delete
+    to_delete: List[Tuple[str]] = []
+
+    # list of mxids to send emails to letting them know the mxid has been deleted
     mxids: List[str] = []
 
     for casefold_address, assoc_tuples in associations.items():
@@ -52,26 +59,23 @@ def update_assosc(self, conn: sqlite3.Connection):
         if len(assoc_tuples) > 1:
             # Iterate over all associations except for the first one, since we've already
             # processed it.
-            for assoc_tuple in assoc_tuples[1:]:
-                to_delete.append([assoc_tuple[0]])
-                mxids.append(assoc_tuple[1])
+            for address, mxid, _ in assoc_tuples[1:]:
+                to_delete.append((address,))
+                mxids.append((mxid, address))
 
     # iterate through the mxids and send email
-    for mxid in mxids:
+    for mxid, address in mxids:
         templateFile = self.sydent.get_branded_template(
             "matrix-org",
             "migration_template.eml",
             ("email", "email.template"),
         )
-        res = cur.execute(
-            "SELECT address FROM local_threepid_associations WHERE mxid = ?", (mxid,)
-        )
-        address = res.fetchone()[0]
+
         sendEmail(
             self.sydent,
             templateFile,
             address,
-            {"mxid": "mxid", "subject_header_value": "MatrixID Deletion"},
+            {"mxid": "mxid", "subject_header_value": "MatrixID Update"},
         )
 
     if len(to_delete) > 0:
@@ -90,46 +94,61 @@ def update_assosc(self, conn: sqlite3.Connection):
 
 
 def update_global_assoc(self, conn: sqlite3.Connection):
+    """Update the DB table global_threepid_associations so that all stored
+    emails are casefolded, the signed association is re-signed and any duplicate
+    mxid's associated with the given email are deleted.
+
+    :return: None
+    """
+
     # get every row where the local server is origin server and medium is email
-    originServer = self.sydent.server_name
+    origin_server = self.sydent.server_name
     medium = "email"
 
     cur = self.sydent.db.cursor()
     res = cur.execute(
         "SELECT address, mxid, sgAssoc FROM global_threepid_associations WHERE medium = ?"
         "AND originServer = ? ORDER BY ts DESC",
-        (medium, originServer),
+        (medium, origin_server),
     )
 
-    associations: Dict[str, List[Tuple[str, str, str]]] = {}
+    # dict that stores email address with mxid, email address, lookup hash, and
+    # signed association
+    associations: Dict[str, List[Tuple[str, str, str, str]]] = {}
 
-    # iterate through selected associations, casefold email, rehash it, and add to
-    # associations dict
-    for row in res.fetchall():
-        casefold_address = row[0].casefold()
+    # iterate through selected associations, casefold email, rehash it, re-sign the
+    # associations and add to associations dict
+    for address, mxid, sg_assoc in res.fetchall():
+        casefold_address = address.casefold()
 
         # rehash the email since hash functions are case-sensitive
         lookup_hash = self.calculate_lookup(casefold_address)
 
-        # update associations with new casefolded address and re-sign
-        sgassoc = json_decoder.decode(row[2])
-        sgassoc["address"] = row[0].casefold()
-        sgAssoc = json.dumps(
+        # update signed associations with new casefolded address and re-sign
+        sg_assoc = json_decoder.decode(sg_assoc)
+        sg_assoc["address"] = address.casefold()
+        sg_assoc = json.dumps(
             signedjson.sign.sign_json(
-                sgassoc, self.sydent.server_name, self.sydent.keyring.ed25519
+                sg_assoc, self.sydent.server_name, self.sydent.keyring.ed25519
             )
         )
 
         if casefold_address in associations:
             associations[casefold_address].append(
-                (row[0], row[1], lookup_hash, sgAssoc)
+                (address, mxid, lookup_hash, sg_assoc)
             )
         else:
-            associations[casefold_address] = [(row[0], row[1], lookup_hash, sgAssoc)]
+            associations[casefold_address] = [(address, mxid, lookup_hash, sg_assoc)]
 
+    # list of arguments to update db with
     db_update_args: List[Tuple[str, str, str, str]] = []
-    to_delete: List[str] = []
-    mxids: List[str] = []
+
+    # list of mxids to delete
+    to_delete: List[Tuple[str]] = []
+
+    # list of mxids and addresses to send emails to letting them know the mxid
+    # has been deleted
+    mxids: List[Tuple[str]] = []
 
     for casefold_address, assoc_tuples in associations.items():
         db_update_args.append(
@@ -145,27 +164,23 @@ def update_global_assoc(self, conn: sqlite3.Connection):
         if len(assoc_tuples) > 1:
             # Iterate over all associations except for the first one, since we've already
             # processed it.
-            for assoc_tuple in assoc_tuples[1:]:
-                to_delete.append([assoc_tuple[0]])
-                mxids.append(assoc_tuple[1])
+            for address, mxid, _, _ in assoc_tuples[1:]:
+                to_delete.append((address,))
+                mxids.append((mxid, address))
 
     # iterate through the mxids and send email
-    for mxid in mxids:
+    for mxid, address in mxids:
         templateFile = self.sydent.get_branded_template(
             "matrix-org",
             "migration_template.eml",
             ("email", "email.template"),
         )
-        res = cur.execute(
-            "SELECT address FROM local_threepid_associations WHERE mxid = ?", (mxid,)
-        )
-        address = res.fetchone()[0]
 
         sendEmail(
             self.sydent,
             templateFile,
             address,
-            {"mxid": "mxid", "subject_header_value": "MatrixID Deletion"},
+            {"mxid": "mxid", "subject_header_value": "MatrixID Update"},
         )
 
     if len(to_delete) > 0:

tests/test_migration.py

@@ -4,7 +4,7 @@
 
 from twisted.trial import unittest
 
-from sydent.db.migration import update_assosc, update_global_assoc
+from sydent.db.migration import update_global_assoc, update_local_associations
 from sydent.util import json_decoder
 from sydent.util.emailutils import sendEmail
 from sydent.util.hash import sha256_and_url_safe_base64
@@ -197,9 +197,12 @@ def test_migration_email(self):
             email_contents = smtp.sendmail.call_args[0][2].decode("utf-8")
             self.assertIn("This is a notification", email_contents)
 
+            # test email was sent
+            smtp.sendmail.assert_called()
+
     def test_local_db_migration(self):
         with patch("sydent.util.emailutils.smtplib"):
-            update_assosc(self, self.sydent.db)
+            update_local_associations(self, self.sydent.db)
 
         cur = self.sydent.db.cursor()
         res = cur.execute("SELECT * FROM local_threepid_associations")