de-duplicate emails received by phabricator multiple times

Summary: this can happen if you have Phabricator and email lists co-mingling such that Phabricator receives an email multiple times. we can prevent this from then spamming everyone or otherwise taking the action multiple times by storing a message id hash and dropping the message if we have more than one message that matches.

Test Plan: simulated sending the same email multiple times on the command line. noted only the first one made it through.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin

Maniphest Tasks: T1726

Differential Revision: https://secure.phabricator.com/D4328

Author: bobtrahan

resources/sql/patches/20130102.metamtareceivedmailmessageidhash.sql

@@ -0,0 +1,3 @@
+ALTER TABLE `{$NAMESPACE}_metamta`.`metamta_receivedmail`
+  ADD `messageIDHash` CHAR(12) BINARY NOT NULL,
+  ADD KEY `key_messageIDHash` (`messageIDHash`);

scripts/mail/mail_handler.php

@@ -34,6 +34,9 @@
   'text' => $text_body,
   'html' => $parser->getMessageBody('html'),
 ));
+$received->setMessageIDHash(
+  PhabricatorHash::digestForIndex($received->getMessageID())
+);
 
 $attachments = array();
 foreach ($parser->getAttachments() as $attachment) {

src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php

@@ -9,6 +9,7 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO {
   protected $relatedPHID;
   protected $authorPHID;
   protected $message;
+  protected $messageIDHash;
 
   public function getConfiguration() {
     return array(
@@ -145,6 +146,27 @@ public function processReceivedMail() {
       return $this->setMessage($message)->save();
     }
 
+    $message_id_hash = $this->getMessageIDHash();
+    if ($message_id_hash) {
+      $messages = $this->loadAllWhere(
+        'messageIDHash = %s',
+        $message_id_hash
+      );
+      $messages_count = count($messages);
+      if ($messages_count > 1) {
+        $first_message = reset($messages);
+        if ($first_message->getID() != $this->getID()) {
+          $message = sprintf(
+            'Ignoring email with message id hash "%s" that has been seen %d '.
+            'times, including this message.',
+            $message_id_hash,
+            $messages_count
+          );
+          return $this->setMessage($message)->save();
+        }
+      }
+    }
+
     list($to,
          $receiver_name,
          $user_id,

src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php

@@ -1076,6 +1076,11 @@ public function getPatches() {
         'type'    => 'sql',
         'name'    => $this->getPatchPath('20130101.confxaction.sql'),
       ),
+      '20130102.metamtareceivedmailmessageidhash.sql' => array(
+        'type'    => 'sql',
+        'name'    =>
+          $this->getPatchPath('20130102.metamtareceivedmailmessageidhash.sql'),
+      ),
     );
   }