Provide a real object ("PhabricatorRepositoryPushEvent") to represent an entire push transaction

Summary:
Ref T4677. Currently, we record individual actions in a push as PhabricatorRepositoryPushLogs, but tie them together only loosely with a `transactionKey`.

Provide a real PushEvent object, and move some of the denormalized fields to it. This primarily just gives us more robust infrastructure for building, e.g., email about pushes, for T4677, since we can act on real PHIDs rather than passing awkward identifiers around.

Test Plan:
  - Performed migration.
  - Looked at database for consistency.
  - Browsed/queried push logs.
  - Pushed a bunch of stuff.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T4677

Differential Revision: https://secure.phabricator.com/D8615

Author: epriestley

resources/sql/autopatches/20140325.push.1.event.sql

@@ -0,0 +1,15 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_pushevent (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  phid VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  repositoryPHID  VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  epoch INT UNSIGNED NOT NULL,
+  pusherPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  remoteAddress INT UNSIGNED,
+  remoteProtocol VARCHAR(32),
+  rejectCode INT UNSIGNED NOT NULL,
+  rejectDetails VARCHAR(64) COLLATE utf8_bin,
+
+  UNIQUE KEY `key_phid` (phid),
+  KEY `key_repository` (repositoryPHID)
+
+) ENGINE=InnoDB, COLLATE=utf8_general_ci;

resources/sql/autopatches/20140325.push.2.eventphid.sql

@@ -0,0 +1,5 @@
+ALTER TABLE {$NAMESPACE}_repository.repository_pushlog
+  ADD pushEventPHID VARCHAR(64) NOT NULL COLLATE utf8_bin AFTER epoch;
+
+ALTER TABLE {$NAMESPACE}_repository.repository_pushlog
+  ADD KEY `key_event` (pushEventPHID);

resources/sql/autopatches/20140325.push.3.groups.php

@@ -0,0 +1,43 @@
+<?php
+
+$conn_w = id(new PhabricatorRepository())->establishConnection('w');
+
+echo "Adding transaction log event groups...\n";
+
+$logs = queryfx_all(
+  $conn_w,
+  'SELECT * FROM %T GROUP BY transactionKey ORDER BY id ASC',
+  'repository_pushlog');
+foreach ($logs as $log) {
+  $id = $log['id'];
+  echo "Migrating log {$id}...\n";
+  if ($log['pushEventPHID']) {
+    continue;
+  }
+
+  $event_phid = id(new PhabricatorRepositoryPushEvent())->generatePHID();
+
+  queryfx(
+    $conn_w,
+    'INSERT INTO %T (phid, repositoryPHID, epoch, pusherPHID, remoteAddress,
+      remoteProtocol, rejectCode, rejectDetails)
+     VALUES (%s, %s, %d, %s, %d, %s, %d, %s)',
+    'repository_pushevent',
+    $event_phid,
+    $log['repositoryPHID'],
+    $log['epoch'],
+    $log['pusherPHID'],
+    $log['remoteAddress'],
+    $log['remoteProtocol'],
+    $log['rejectCode'],
+    $log['rejectDetails']);
+
+  queryfx(
+    $conn_w,
+    'UPDATE %T SET pushEventPHID = %s WHERE transactionKey = %s',
+    'repository_pushlog',
+    $event_phid,
+    $log['transactionKey']);
+}
+
+echo "Done.\n";

resources/sql/autopatches/20140325.push.4.prune.sql

@@ -0,0 +1,14 @@
+ALTER TABLE {$NAMESPACE}_repository.repository_pushlog
+  DROP remoteAddress;
+
+ALTER TABLE {$NAMESPACE}_repository.repository_pushlog
+  DROP remoteProtocol;
+
+ALTER TABLE {$NAMESPACE}_repository.repository_pushlog
+  DROP transactionKey;
+
+ALTER TABLE {$NAMESPACE}_repository.repository_pushlog
+  DROP rejectCode;
+
+ALTER TABLE {$NAMESPACE}_repository.repository_pushlog
+  DROP rejectDetails;

src/__phutil_library_map__.php

@@ -1958,11 +1958,14 @@
     'PhabricatorRepositoryPHIDTypeArcanistProject' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeArcanistProject.php',
     'PhabricatorRepositoryPHIDTypeCommit' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeCommit.php',
     'PhabricatorRepositoryPHIDTypeMirror' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeMirror.php',
+    'PhabricatorRepositoryPHIDTypePushEvent' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypePushEvent.php',
     'PhabricatorRepositoryPHIDTypePushLog' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypePushLog.php',
     'PhabricatorRepositoryPHIDTypeRepository' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeRepository.php',
     'PhabricatorRepositoryParsedChange' => 'applications/repository/data/PhabricatorRepositoryParsedChange.php',
     'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php',
     'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php',
+    'PhabricatorRepositoryPushEvent' => 'applications/repository/storage/PhabricatorRepositoryPushEvent.php',
+    'PhabricatorRepositoryPushEventQuery' => 'applications/repository/query/PhabricatorRepositoryPushEventQuery.php',
     'PhabricatorRepositoryPushLog' => 'applications/repository/storage/PhabricatorRepositoryPushLog.php',
     'PhabricatorRepositoryPushLogQuery' => 'applications/repository/query/PhabricatorRepositoryPushLogQuery.php',
     'PhabricatorRepositoryPushLogSearchEngine' => 'applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php',
@@ -4792,11 +4795,18 @@
     'PhabricatorRepositoryPHIDTypeArcanistProject' => 'PhabricatorPHIDType',
     'PhabricatorRepositoryPHIDTypeCommit' => 'PhabricatorPHIDType',
     'PhabricatorRepositoryPHIDTypeMirror' => 'PhabricatorPHIDType',
+    'PhabricatorRepositoryPHIDTypePushEvent' => 'PhabricatorPHIDType',
     'PhabricatorRepositoryPHIDTypePushLog' => 'PhabricatorPHIDType',
     'PhabricatorRepositoryPHIDTypeRepository' => 'PhabricatorPHIDType',
     'PhabricatorRepositoryParsedChange' => 'Phobject',
     'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine',
     'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
+    'PhabricatorRepositoryPushEvent' =>
+    array(
+      0 => 'PhabricatorRepositoryDAO',
+      1 => 'PhabricatorPolicyInterface',
+    ),
+    'PhabricatorRepositoryPushEventQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorRepositoryPushLog' =>
     array(
       0 => 'PhabricatorRepositoryDAO',

src/applications/diffusion/controller/DiffusionPushLogListController.php

@@ -52,14 +52,15 @@ public function renderResultsList(
       // Reveal this if it's valid and the user can edit the repository.
       $remote_addr = '-';
       if (isset($editable_repos[$log->getRepositoryPHID()])) {
-        $remote_long = $log->getRemoteAddress();
+        $remote_long = $log->getPushEvent()->getRemoteAddress();
         if ($remote_long) {
           $remote_addr = long2ip($remote_long);
         }
       }
 
       $callsign = $log->getRepository()->getCallsign();
       $rows[] = array(
+        $log->getPushEvent()->getID(),
         phutil_tag(
           'a',
           array(
@@ -68,7 +69,7 @@ public function renderResultsList(
           $callsign),
         $this->getHandle($log->getPusherPHID())->renderLink(),
         $remote_addr,
-        $log->getRemoteProtocol(),
+        $log->getPushEvent()->getRemoteProtocol(),
         $log->getRefType(),
         $log->getRefName(),
         phutil_tag(
@@ -86,14 +87,15 @@ public function renderResultsList(
 
         // TODO: Make these human-readable.
         $log->getChangeFlags(),
-        $log->getRejectCode(),
+        $log->getPushEvent()->getRejectCode(),
         phabricator_datetime($log->getEpoch(), $viewer),
       );
     }
 
     $table = id(new AphrontTableView($rows))
       ->setHeaders(
         array(
+          pht('Push'),
           pht('Repository'),
           pht('Pusher'),
           pht('From'),
@@ -113,6 +115,7 @@ public function renderResultsList(
           '',
           '',
           '',
+          '',
           'wide',
           'n',
           'n',

src/applications/diffusion/engine/DiffusionCommitHookEngine.php

@@ -30,6 +30,8 @@ final class DiffusionCommitHookEngine extends Phobject {
   private $gitCommits = array();
 
   private $heraldViewerProjects;
+  private $rejectCode = PhabricatorRepositoryPushLog::REJECT_BROKEN;
+  private $rejectDetails;
 
 
 /* -(  Config  )------------------------------------------------------------- */
@@ -62,14 +64,6 @@ private function getRemoteAddressForLog() {
     return $remote_address;
   }
 
-  private function getTransactionKey() {
-    if (!$this->transactionKey) {
-      $entropy = Filesystem::readRandomBytes(64);
-      $this->transactionKey = PhabricatorHash::digestForIndex($entropy);
-    }
-    return $this->transactionKey;
-  }
-
   public function setSubversionTransactionInfo($transaction, $repository) {
     $this->subversionTransaction = $transaction;
     $this->subversionRepository = $repository;
@@ -137,10 +131,7 @@ public function execute() {
       } catch (DiffusionCommitHookRejectException $ex) {
         // If we're rejecting dangerous changes, flag everything that we've
         // seen as rejected so it's clear that none of it was accepted.
-        foreach ($all_updates as $update) {
-          $update->setRejectCode(
-            PhabricatorRepositoryPushLog::REJECT_DANGEROUS);
-        }
+        $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_DANGEROUS;
         throw $ex;
       }
 
@@ -156,19 +147,26 @@ public function execute() {
 
       // If we make it this far, we're accepting these changes. Mark all the
       // logs as accepted.
-      foreach ($all_updates as $update) {
-        $update->setRejectCode(PhabricatorRepositoryPushLog::REJECT_ACCEPT);
-      }
+      $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_ACCEPT;
     } catch (Exception $ex) {
       // We'll throw this again in a minute, but we want to save all the logs
       // first.
       $caught = $ex;
     }
 
     // Save all the logs no matter what the outcome was.
-    foreach ($all_updates as $update) {
-      $update->save();
-    }
+    $event = $this->newPushEvent();
+
+    $event->setRejectCode($this->rejectCode);
+    $event->setRejectDetails($this->rejectDetails);
+
+    $event->openTransaction();
+      $event->save();
+      foreach ($all_updates as $update) {
+        $update->setPushEventPHID($event->getPHID());
+        $update->save();
+      }
+    $event->saveTransaction();
 
     if ($caught) {
       throw $caught;
@@ -296,10 +294,8 @@ private function applyHeraldRules(
     }
 
     if ($blocking_effect) {
-      foreach ($all_updates as $update) {
-        $update->setRejectCode(PhabricatorRepositoryPushLog::REJECT_HERALD);
-        $update->setRejectDetails($blocking_effect->getRulePHID());
-      }
+      $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_HERALD;
+      $this->rejectDetails = $blocking_effect->getRulePHID();
 
       $message = $blocking_effect->getTarget();
       if (!strlen($message)) {
@@ -596,12 +592,8 @@ private function applyCustomHooks(array $updates) {
           continue;
         }
 
-        // Mark everything as rejected by this hook.
-        foreach ($updates as $update) {
-          $update->setRejectCode(
-            PhabricatorRepositoryPushLog::REJECT_EXTERNAL);
-          $update->setRejectDetails(basename($hook));
-        }
+        $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_EXTERNAL;
+        $this->rejectDetails = basename($hook);
 
         throw new DiffusionCommitHookRejectException(
           pht(
@@ -983,24 +975,23 @@ private function findSubversionContentUpdates(array $ref_updates) {
 
 
   private function newPushLog() {
-    // NOTE: By default, we create these with REJECT_BROKEN as the reject
-    // code. This indicates a broken hook, and covers the case where we
-    // encounter some unexpected exception and consequently reject the changes.
-
     // NOTE: We generate PHIDs up front so the Herald transcripts can pick them
     // up.
     $phid = id(new PhabricatorRepositoryPushLog())->generatePHID();
 
     return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer())
       ->setPHID($phid)
-      ->attachRepository($this->getRepository())
       ->setRepositoryPHID($this->getRepository()->getPHID())
-      ->setEpoch(time())
+      ->setEpoch(time());
+  }
+
+  private function newPushEvent() {
+    $viewer = $this->getViewer();
+    return PhabricatorRepositoryPushEvent::initializeNewEvent($viewer)
+      ->setRepositoryPHID($this->getRepository()->getPHID())
       ->setRemoteAddress($this->getRemoteAddressForLog())
       ->setRemoteProtocol($this->getRemoteProtocol())
-      ->setTransactionKey($this->getTransactionKey())
-      ->setRejectCode(PhabricatorRepositoryPushLog::REJECT_BROKEN)
-      ->setRejectDetails(null);
+      ->setEpoch(time());
   }
 
   public function loadChangesetsForCommit($identifier) {

src/applications/repository/phid/PhabricatorRepositoryPHIDTypePushEvent.php

@@ -0,0 +1,40 @@
+<?php
+
+final class PhabricatorRepositoryPHIDTypePushEvent
+  extends PhabricatorPHIDType {
+
+  const TYPECONST = 'PSHE';
+
+  public function getTypeConstant() {
+    return self::TYPECONST;
+  }
+
+  public function getTypeName() {
+    return pht('Push Event');
+  }
+
+  public function newObject() {
+    return new PhabricatorRepositoryPushEvent();
+  }
+
+  protected function buildQueryForObjects(
+    PhabricatorObjectQuery $query,
+    array $phids) {
+
+    return id(new PhabricatorRepositoryPushEventQuery())
+      ->withPHIDs($phids);
+  }
+
+  public function loadHandles(
+    PhabricatorHandleQuery $query,
+    array $handles,
+    array $objects) {
+
+    foreach ($handles as $phid => $handle) {
+      $event = $objects[$phid];
+
+      $handle->setName(pht('Push Event %d', $event->getID()));
+    }
+  }
+
+}