Casefold when processing email addresses

changelog.d/374.misc

@@ -0,0 +1 @@
+Implements casefolding on email addresses when binding to mxid, storing associations, or looking up asssociations by email. 

res/matrix-org/migration_template.eml

@@ -0,0 +1,38 @@
+Date: %(date)s
+From: %(from)s
+To: %(to)s
+Message-ID: %(messageid)s
+Subject: %(subject_header_value)s
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+  boundary="%(multipart_boundary)s"
+
+--%(multipart_boundary)s
+Content-Type: text/plain; charset=UTF-8
+Content-Disposition: inline
+
+Hello,
+
+In the past, identity services did not take case into account when creating and storing
+mxids. This led to your email address, %(to)s being stored with two mxids. A recent update
+changed this behavior and streamlined the mxids associated with your email address.
+This is a notification to let you know that matrix id %(mxid)s has been deleted in accordance
+with this update.
+
+
+About Matrix:
+
+Matrix.org is an open standard for interoperable, decentralised, real-time communication
+over IP, supporting group chat, file transfer, voice and video calling, integrations to
+other apps, bridges to other communication systems and much more. It can be used to power
+Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere
+you need a standard HTTP API for publishing and subscribing to data whilst tracking the
+conversation history.
+
+Matrix defines the standard, and provides open source reference implementations of
+Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you
+create new communication solutions or extend the capabilities and reach of existing ones.
+
+Thanks,
+
+Matrix

sydent/db/invite_tokens.py

@@ -34,6 +34,9 @@ def storeToken(
         :param sender: The MXID of the user that sent the invite.
         :param token: The token to store.
         """
+        if medium == "email":
+            address = address.casefold()
+
         cur = self.sydent.db.cursor()
 
         cur.execute(

sydent/db/migration.py

@@ -0,0 +1,197 @@
+import json
+import sqlite3
+
+import signedjson.sign
+
+from sydent.util import json_decoder
+from sydent.util.emailutils import sendEmail
+
+
+def update_local_associations(self, conn: sqlite3.Connection):
+    """Update the DB table local_threepid_associations so that all stored
+    emails are casefolded, and any duplicate mxid's associated with the
+    given email are deleted.
+
+    :return: None
+    """
+    cur = conn.cursor()
+
+    res = cur.execute(
+        "SELECT address, mxid FROM local_threepid_associations WHERE medium = 'email'"
+        "ORDER BY ts DESC"
+    )
+
+    # a dict that associates an email address with correspoinding mxids and lookup hashes
+    associations: Dict[str, List[Tuple[str, str, str]]] = {}
+
+    # iterate through selected associations, casefold email, rehash it, and add to
+    # associations dict
+    for address, mxid in res.fetchall():
+        casefold_address = address.casefold()
+
+        # rehash email since hashes are case-sensitive
+        lookup_hash = self.calculate_lookup(casefold_address)
+
+        if casefold_address in associations:
+            associations[casefold_address].append((address, mxid, lookup_hash))
+        else:
+            associations[casefold_address] = [(address, mxid, lookup_hash)]
+
+    # list of arguments to update db with
+    db_update_args: List[Tuple[str, str, str, str]] = []
+
+    # list of mxids to delete
+    to_delete: List[Tuple[str]] = []
+
+    # list of mxids to send emails to letting them know the mxid has been deleted
+    mxids: List[str] = []
+
+    for casefold_address, assoc_tuples in associations.items():
+        db_update_args.append(
+            (
+                casefold_address,
+                assoc_tuples[0][2],
+                assoc_tuples[0][0],
+                assoc_tuples[0][1],
+            )
+        )
+
+        if len(assoc_tuples) > 1:
+            # Iterate over all associations except for the first one, since we've already
+            # processed it.
+            for address, mxid, _ in assoc_tuples[1:]:
+                to_delete.append((address,))
+                mxids.append((mxid, address))
+
+    # iterate through the mxids and send email
+    for mxid, address in mxids:
+        templateFile = self.sydent.get_branded_template(
+            "matrix-org",
+            "migration_template.eml",
+            ("email", "email.template"),
+        )
+
+        sendEmail(
+            self.sydent,
+            templateFile,
+            address,
+            {"mxid": "mxid", "subject_header_value": "MatrixID Update"},
+        )
+
+    if len(to_delete) > 0:
+        cur.executemany(
+            "DELETE FROM local_threepid_associations WHERE address = ?", to_delete
+        )
+
+    if len(db_update_args) > 0:
+        cur.executemany(
+            "UPDATE local_threepid_associations SET address = ?, lookup_hash = ? WHERE address = ? AND mxid = ?",
+            db_update_args,
+        )
+
+    # We've finished updating the database, committing the transaction.
+    conn.commit()
+
+
+def update_global_assoc(self, conn: sqlite3.Connection):
+    """Update the DB table global_threepid_associations so that all stored
+    emails are casefolded, the signed association is re-signed and any duplicate
+    mxid's associated with the given email are deleted.
+
+    :return: None
+    """
+
+    # get every row where the local server is origin server and medium is email
+    origin_server = self.sydent.server_name
+    medium = "email"
+
+    cur = self.sydent.db.cursor()
+    res = cur.execute(
+        "SELECT address, mxid, sgAssoc FROM global_threepid_associations WHERE medium = ?"
+        "AND originServer = ? ORDER BY ts DESC",
+        (medium, origin_server),
+    )
+
+    # dict that stores email address with mxid, email address, lookup hash, and
+    # signed association
+    associations: Dict[str, List[Tuple[str, str, str, str]]] = {}
+
+    # iterate through selected associations, casefold email, rehash it, re-sign the
+    # associations and add to associations dict
+    for address, mxid, sg_assoc in res.fetchall():
+        casefold_address = address.casefold()
+
+        # rehash the email since hash functions are case-sensitive
+        lookup_hash = self.calculate_lookup(casefold_address)
+
+        # update signed associations with new casefolded address and re-sign
+        sg_assoc = json_decoder.decode(sg_assoc)
+        sg_assoc["address"] = address.casefold()
+        sg_assoc = json.dumps(
+            signedjson.sign.sign_json(
+                sg_assoc, self.sydent.server_name, self.sydent.keyring.ed25519
+            )
+        )
+
+        if casefold_address in associations:
+            associations[casefold_address].append(
+                (address, mxid, lookup_hash, sg_assoc)
+            )
+        else:
+            associations[casefold_address] = [(address, mxid, lookup_hash, sg_assoc)]
+
+    # list of arguments to update db with
+    db_update_args: List[Tuple[str, str, str, str]] = []
+
+    # list of mxids to delete
+    to_delete: List[Tuple[str]] = []
+
+    # list of mxids and addresses to send emails to letting them know the mxid
+    # has been deleted
+    mxids: List[Tuple[str]] = []
+
+    for casefold_address, assoc_tuples in associations.items():
+        db_update_args.append(
+            (
+                casefold_address,
+                assoc_tuples[0][2],
+                assoc_tuples[0][3],
+                assoc_tuples[0][0],
+                assoc_tuples[0][1],
+            )
+        )
+
+        if len(assoc_tuples) > 1:
+            # Iterate over all associations except for the first one, since we've already
+            # processed it.
+            for address, mxid, _, _ in assoc_tuples[1:]:
+                to_delete.append((address,))
+                mxids.append((mxid, address))
+
+    # iterate through the mxids and send email
+    for mxid, address in mxids:
+        templateFile = self.sydent.get_branded_template(
+            "matrix-org",
+            "migration_template.eml",
+            ("email", "email.template"),
+        )
+
+        sendEmail(
+            self.sydent,
+            templateFile,
+            address,
+            {"mxid": "mxid", "subject_header_value": "MatrixID Update"},
+        )
+
+    if len(to_delete) > 0:
+        cur.executemany(
+            "DELETE FROM global_threepid_associations WHERE address = ?", to_delete
+        )
+
+    if len(db_update_args) > 0:
+        cur.executemany(
+            "UPDATE global_threepid_associations SET address = ?, lookup_hash = ?, sgAssoc = ? WHERE address = ? AND mxid = ?",
+            db_update_args,
+        )
+
+    conn.commit()

sydent/db/threepid_associations.py

@@ -189,6 +189,7 @@ def signedAssociationStringForThreepid(
         :return: The signed association, or None if no association was found for this
             3PID.
         """
+
         cur = self.sydent.db.cursor()
         # We treat address as case-insensitive because that's true for all the
         # threepids we have currently (we treat the local part of email addresses as
@@ -219,6 +220,9 @@ def getMxid(self, medium: str, address: str) -> Optional[str]:
 
         :return: The associated MXID, or None if no MXID is associated with this 3PID.
         """
+        if medium == "email":
+            address = address.casefold()
+
         cur = self.sydent.db.cursor()
         res = cur.execute(
             "select mxid from global_threepid_associations where "
@@ -307,6 +311,9 @@ def addAssociation(
         :param commit: Whether to commit the database transaction after inserting the
             association.
         """
+        if assoc.medium == "email":
+            assoc.address = assoc.address.casefold()
+
         cur = self.sydent.db.cursor()
         cur.execute(
             "insert or ignore into global_threepid_associations "
@@ -357,6 +364,9 @@ def removeAssociation(self, medium: str, address: str) -> None:
         :param medium: The medium for the 3PID.
         :param address: The address for the 3PID.
         """
+        if medium == "email":
+            address = address.casefold()
+
         cur = self.sydent.db.cursor()
         cur.execute(
             "DELETE FROM global_threepid_associations WHERE "

sydent/terms/terms.py

@@ -107,7 +107,7 @@ def get_terms(sydent) -> Optional[Terms]:
             return Terms(None)
 
         with open(termsPath) as fp:
-            termsYaml = yaml.full_load(fp)
+            termsYaml = yaml.safe_load(fp)
         if "master_version" not in termsYaml:
             raise Exception("No master version")
         if "docs" not in termsYaml:

sydent/threepid/bind.py

@@ -58,6 +58,9 @@ def addBinding(self, medium: str, address: str, mxid: str) -> Dict[str, Any]:
 
         :return: The signed association.
         """
+        if medium == "email":
+            address = address.casefold()
+
         localAssocStore = LocalAssociationStore(self.sydent)
 
         # Fill out the association details