Add an assocations-like "Edges" framework

epriestley · epriestley · commit 877cb136e856 · 2012-04-04T15:30:21.000-07:00
Summary: We have a lot of cases where we store object relationships, but it's all kind of messy and custom. Some particular problems: - We go to great lengths to enforce order stability in Differential revisions, but the implementation is complex and inelegant. - Some relationships are stored on-object, so we can't pull the inverses easily. For example, Maniphest shows child tasks but not parent tasks. - I want to add more of these and don't want to continue building custom stuff. - UIs like the "attach stuff to other stuff" UI need custom branches for each object type. - Stuff like "allow commits to close tasks" is notrivial because of nonstandard metadata storage. Provide an association-like "edge" framework to fix these problems. This is nearly identical to associations, with a few differences: - I put edge metadata in a separate table and don't load it by default, to keep edge rows small and allow large metadata if necessary. The on-edge metadata seemed to get abused a lot at Facebook. - I put a 'seq' column on the edges to ensure they have an explicit, stable ordering within a source and type. This isn't actually used anywhere yet, but my first target is attaching commits to tasks for T904. Test Plan: Made a mock page that used Editor and Query. Verified adding and removing edges, overwriting edges, writing and loading edge data, sequence number generation. Reviewers: btrahan Reviewed By: btrahan CC: aran, 20after4 Differential Revision: https://secure.phabricator.com/D2088
diff --git a/resources/sql/patches/126.edges.sql b/resources/sql/patches/126.edges.sql
@@ -0,0 +1,123 @@
+CREATE TABLE phabricator_maniphest.edge (
+  src VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  type VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dst VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dateCreated INT UNSIGNED NOT NULL,
+  seq INT UNSIGNED NOT NULL,
+  dataID INT UNSIGNED,
+  PRIMARY KEY (src, type, dst),
+  KEY (src, type, dateCreated, seq)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+CREATE TABLE phabricator_maniphest.edgedata (
+  id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  data LONGTEXT NOT NULL COLLATE utf8_bin
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+
+
+CREATE TABLE phabricator_repository.edge (
+  src VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  type VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dst VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dateCreated INT UNSIGNED NOT NULL,
+  seq INT UNSIGNED NOT NULL,
+  dataID INT UNSIGNED,
+  PRIMARY KEY (src, type, dst),
+  KEY (src, type, dateCreated, seq)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+CREATE TABLE phabricator_repository.edgedata (
+  id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  data LONGTEXT NOT NULL COLLATE utf8_bin
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+
+
+CREATE TABLE phabricator_differential.edge (
+  src VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  type VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dst VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dateCreated INT UNSIGNED NOT NULL,
+  seq INT UNSIGNED NOT NULL,
+  dataID INT UNSIGNED,
+  PRIMARY KEY (src, type, dst),
+  KEY (src, type, dateCreated, seq)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+CREATE TABLE phabricator_differential.edgedata (
+  id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  data LONGTEXT NOT NULL COLLATE utf8_bin
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+
+
+CREATE TABLE phabricator_file.edge (
+  src VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  type VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dst VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dateCreated INT UNSIGNED NOT NULL,
+  seq INT UNSIGNED NOT NULL,
+  dataID INT UNSIGNED,
+  PRIMARY KEY (src, type, dst),
+  KEY (src, type, dateCreated, seq)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+CREATE TABLE phabricator_file.edgedata (
+  id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  data LONGTEXT NOT NULL COLLATE utf8_bin
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+
+
+CREATE TABLE phabricator_user.edge (
+  src VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  type VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dst VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dateCreated INT UNSIGNED NOT NULL,
+  seq INT UNSIGNED NOT NULL,
+  dataID INT UNSIGNED,
+  PRIMARY KEY (src, type, dst),
+  KEY (src, type, dateCreated, seq)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+CREATE TABLE phabricator_user.edgedata (
+  id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  data LONGTEXT NOT NULL COLLATE utf8_bin
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+
+
+CREATE TABLE phabricator_project.edge (
+  src VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  type VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dst VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dateCreated INT UNSIGNED NOT NULL,
+  seq INT UNSIGNED NOT NULL,
+  dataID INT UNSIGNED,
+  PRIMARY KEY (src, type, dst),
+  KEY (src, type, dateCreated, seq)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+CREATE TABLE phabricator_project.edgedata (
+  id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  data LONGTEXT NOT NULL COLLATE utf8_bin
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+
+
+CREATE TABLE phabricator_metamta.edge (
+  src VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  type VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dst VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dateCreated INT UNSIGNED NOT NULL,
+  seq INT UNSIGNED NOT NULL,
+  dataID INT UNSIGNED,
+  PRIMARY KEY (src, type, dst),
+  KEY (src, type, dateCreated, seq)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
+
+CREATE TABLE phabricator_metamta.edgedata (
+  id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  data LONGTEXT NOT NULL COLLATE utf8_bin
+) ENGINE=InnoDB, COLLATE utf8_general_ci;
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
@@ -552,6 +552,10 @@
     'PhabricatorDisabledUserController' => 'applications/auth/controller/disabled',
     'PhabricatorDraft' => 'applications/draft/storage/draft',
     'PhabricatorDraftDAO' => 'applications/draft/storage/base',
+    'PhabricatorEdgeConfig' => 'infrastructure/edges/constants/config',
+    'PhabricatorEdgeConstants' => 'infrastructure/edges/constants/base',
+    'PhabricatorEdgeEditor' => 'infrastructure/edges/editor/edge',
+    'PhabricatorEdgeQuery' => 'infrastructure/edges/query/edge',
     'PhabricatorEmailLoginController' => 'applications/auth/controller/email',
     'PhabricatorEmailTokenController' => 'applications/auth/controller/emailtoken',
     'PhabricatorEnv' => 'infrastructure/env',
@@ -1403,6 +1407,8 @@
     'PhabricatorDisabledUserController' => 'PhabricatorAuthController',
     'PhabricatorDraft' => 'PhabricatorDraftDAO',
     'PhabricatorDraftDAO' => 'PhabricatorLiskDAO',
+    'PhabricatorEdgeConfig' => 'PhabricatorEdgeConstants',
+    'PhabricatorEdgeQuery' => 'PhabricatorQuery',
     'PhabricatorEmailLoginController' => 'PhabricatorAuthController',
     'PhabricatorEmailTokenController' => 'PhabricatorAuthController',
     'PhabricatorEnvTestCase' => 'PhabricatorTestCase',
diff --git a/src/applications/base/storage/lisk/PhabricatorLiskDAO.php b/src/applications/base/storage/lisk/PhabricatorLiskDAO.php
@@ -1,7 +1,7 @@
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,9 +16,55 @@
  * limitations under the License.
  */
 
-
+/**
+ * @task edges  Managing Edges
+ * @task config Configuring Storage
+ */
 abstract class PhabricatorLiskDAO extends LiskDAO {
 
+  private $edges = array();
+
+
+/* -(  Managing Edges  )----------------------------------------------------- */
+
+
+  /**
+   * @task edges
+   */
+  public function attachEdges(array $edges) {
+    foreach ($edges as $type => $type_edges) {
+      $this->edges[$type] = $type_edges;
+    }
+    return $this;
+  }
+
+
+  /**
+   * @task edges
+   */
+  public function getEdges($type) {
+    $edges = idx($this->edges, $type);
+    if ($edges === null) {
+      throw new Exception("Call attachEdges() before getEdges()!");
+    }
+    return $edges;
+  }
+
+
+  /**
+   * @task edges
+   */
+  public function getEdgePHIDs($type) {
+    return ipull($this->getEdges($type), 'dst');
+  }
+
+
+/* -(  Configuring Storage  )------------------------------------------------ */
+
+
+  /**
+   * @task config
+   */
   public function establishLiveConnection($mode) {
     $conf_provider = PhabricatorEnv::getEnvConfig(
       'mysql.configuration_provider', 'DatabaseConfigurationProvider');
@@ -34,6 +80,9 @@ public function establishLiveConnection($mode) {
       ));
   }
 
+  /**
+   * @task config
+   */
   public function getTableName() {
     $str = 'phabricator';
     $len = strlen($str);
@@ -54,5 +103,8 @@ public function getTableName() {
     }
   }
 
+  /**
+   * @task config
+   */
   abstract public function getApplicationName();
 }
diff --git a/src/docs/developer/using_edges.diviner b/src/docs/developer/using_edges.diviner
@@ -0,0 +1,31 @@
+@title Using Edges
+@group developer
+
+Guide to the Edges infrastructure.
+
+= Overview =
+
+Edges are a generic way of storing a relationship between two objects (like
+a Task and its attached files). If you are familiar with the Facebook
+associations framework, Phabricator Edges are substantially similar.
+
+An edge is defined by a source PHID (the edge origin), a destination PHID
+(the edge destination) and an edge type (which describes the relationship,
+like "is subscribed to" or "has attached file").
+
+Every edge is directional, and stored alongside the source object. Some edges
+are configured to automatically write an inverse edge, effectively building
+a bidirectional relationship. The strength of storing relationships like this
+is that they work when databases are partitioned or sharded.
+
+= Reading Edges =
+
+You can load edges with @{class:PhabricatorEdgeQuery}.
+
+= Writing Edges =
+
+You can edit edges with @{class:PhabricatorEdgeEditor}.
+
+= Edges and Lisk =
+
+@{class:PhabricatorLiskDAO} includes some builtin support for edges.
diff --git a/src/infrastructure/edges/constants/base/PhabricatorEdgeConstants.php b/src/infrastructure/edges/constants/base/PhabricatorEdgeConstants.php
@@ -0,0 +1,21 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+abstract class PhabricatorEdgeConstants {
+
+}
diff --git a/src/infrastructure/edges/constants/base/__init__.php b/src/infrastructure/edges/constants/base/__init__.php
@@ -0,0 +1,10 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+
+phutil_require_source('PhabricatorEdgeConstants.php');
diff --git a/src/infrastructure/edges/constants/config/PhabricatorEdgeConfig.php b/src/infrastructure/edges/constants/config/PhabricatorEdgeConfig.php
@@ -0,0 +1,59 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorEdgeConfig extends PhabricatorEdgeConstants {
+
+  const TABLE_NAME_EDGE       = 'edge';
+  const TABLE_NAME_EDGEDATA   = 'edgedata';
+
+  const TYPE_TASK_HAS_COMMIT  = 1;
+  const TYPE_COMMIT_HAS_TASK  = 2;
+
+  public static function getInverse($edge_type) {
+    static $map = array(
+      self::TYPE_TASK_HAS_COMMIT => self::TYPE_COMMIT_HAS_TASK,
+      self::TYPE_COMMIT_HAS_TASK => self::TYPE_TASK_HAS_COMMIT,
+    );
+
+    return idx($map, $edge_type);
+  }
+
+  public static function establishConnection($phid_type, $conn_type) {
+    static $class_map = array(
+      PhabricatorPHIDConstants::PHID_TYPE_TASK  => 'ManiphestTask',
+      PhabricatorPHIDConstants::PHID_TYPE_CMIT  => 'PhabricatorRepository',
+      PhabricatorPHIDConstants::PHID_TYPE_DREV  => 'DifferentialRevision',
+      PhabricatorPHIDConstants::PHID_TYPE_FILE  => 'PhabricatorFile',
+      PhabricatorPHIDConstants::PHID_TYPE_USER  => 'PhabricatorUser',
+      PhabricatorPHIDConstants::PHID_TYPE_PROJ  => 'PhabricatorProject',
+      PhabricatorPHIDConstants::PHID_TYPE_MLST  =>
+        'PhabricatorMetaMTAMailingList',
+    );
+
+    $class = idx($class_map, $phid_type);
+
+    if (!$class) {
+      throw new Exception(
+        "Edges are not available for objects of type '{$phid_type}'!");
+    }
+
+    return newv($class, array())->establishConnection($conn_type);
+  }
+
+
+}
diff --git a/src/infrastructure/edges/constants/config/__init__.php b/src/infrastructure/edges/constants/config/__init__.php
@@ -0,0 +1,15 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/phid/constants');
+phutil_require_module('phabricator', 'infrastructure/edges/constants/base');
+
+phutil_require_module('phutil', 'utils');
+
+
+phutil_require_source('PhabricatorEdgeConfig.php');
diff --git a/src/infrastructure/edges/editor/edge/PhabricatorEdgeEditor.php b/src/infrastructure/edges/editor/edge/PhabricatorEdgeEditor.php
diff --git a/src/infrastructure/edges/editor/edge/__init__.php b/src/infrastructure/edges/editor/edge/__init__.php
diff --git a/src/infrastructure/edges/query/edge/PhabricatorEdgeQuery.php b/src/infrastructure/edges/query/edge/PhabricatorEdgeQuery.php
diff --git a/src/infrastructure/edges/query/edge/__init__.php b/src/infrastructure/edges/query/edge/__init__.php
