Simplify daemon management: "phd start"

Summary:
  - Merge CommitTask daemon into PullLocal daemon. This is another artifact of past instability (and order-dependent parsers). We still publish to the timeline, although this was the last consumer. Long term we'll probably delete timeline and move to webhooks, since everyone who has asked about this stuff has been eager to trade away the durability and ordering of the timeline for the ease of use of webhooks. There's also no reason to timeline this anymore since parsing is no longer order-dependent.
  - Add `phd start` to start all the daemons you need. Add `phd restart` to restart all the daemons you need. So cool~
  - Simplify and improve phd and Diffusion daemon documentation.

Test Plan:
  - Ran `phd start`.
  - Ran `phd restart`.
  - Generated/read documentation.
  - Imported some stuff, got clean parses.

Reviewers: btrahan, csilvers

Reviewed By: csilvers

CC: aran, jungejason, nh

Differential Revision: https://secure.phabricator.com/D2433

Author: epriestley

conf/default.conf.php

@@ -937,6 +937,11 @@
   // track running daemons.
   'phd.pid-directory' => '/var/tmp/phd',
 
+  // Number of "TaskMaster" daemons that "phd start" should start. You can
+  // raise this if you have a task backlog, or explicitly launch more with
+  // "phd launch <N> taskmaster".
+  'phd.start-taskmasters' => 4,
+
   // This value is an input to the hash function when building resource hashes.
   // It has no security value, but if you accidentally poison user caches (by
   // pushing a bad patch or having something go wrong with a CDN, e.g.) you can

scripts/daemon/phabricator_daemon_launcher.php

@@ -34,7 +34,8 @@ function must_have_extension($ext) {
   }
 }
 
-switch (isset($argv[1]) ? $argv[1] : 'help') {
+$command = isset($argv[1]) ? $argv[1] : 'help';
+switch ($command) {
   case 'list':
     $err = $control->executeListCommand();
     exit($err);
@@ -48,7 +49,45 @@ function must_have_extension($ext) {
     $err = $control->executeStopCommand($pass_argv);
     exit($err);
 
-  case 'repository-launch-readonly':
+  case 'restart':
+    $err = $control->executeStopCommand(array());
+    if ($err) {
+      exit($err);
+    }
+    /* Fall Through */
+  case 'start':
+    $running = $control->loadRunningDaemons();
+    if ($running) {
+      echo phutil_console_wrap(
+        "phd start: Unable to start daemons because daemons are already ".
+        "running.\n".
+        "You can view running daemons with 'phd list'.\n".
+        "You can stop running daemons with 'phd stop'.\n".
+        "You can use 'phd restart' to stop all daemons before starting new ".
+        "daemons.\n");
+      exit(1);
+    }
+
+    $daemons = array(
+      array('PhabricatorRepositoryPullLocalDaemon', array()),
+      array('PhabricatorGarbageCollectorDaemon', array()),
+    );
+
+    $taskmasters = PhabricatorEnv::getEnvConfig('phd.start-taskmasters');
+    for ($ii = 0; $ii < $taskmasters; $ii++) {
+      $daemons[] = array('PhabricatorTaskmasterDaemon', array());
+    }
+
+    will_launch($control);
+    foreach ($daemons as $spec) {
+      list($name, $argv) = $spec;
+      echo "Launching '{$name}'...\n";
+      $control->launchDaemon($name, $argv);
+    }
+
+    echo "Done.\n";
+    break;
+
     $need_launch = phd_load_tracked_repositories();
     if (!$need_launch) {
       echo "There are no repositories with tracking enabled.\n";
@@ -68,7 +107,16 @@ function must_have_extension($ext) {
     echo "Done.\n";
     break;
 
+  case 'repository-launch-readonly':
   case 'repository-launch-master':
+    if ($command == 'repository-launch-readonly') {
+      $daemon_args = array(
+        '--no-discovery',
+      );
+    } else {
+      $daemon_args = array();
+    }
+
     $need_launch = phd_load_tracked_repositories();
     if (!$need_launch) {
       echo "There are no repositories with tracking enabled.\n";
@@ -77,18 +125,12 @@ function must_have_extension($ext) {
 
     will_launch($control);
 
-    echo "Launching PullLocal daemon in master mode...\n";
+    echo "Launching PullLocal daemon...\n";
     $control->launchDaemon(
       'PhabricatorRepositoryPullLocalDaemon',
-      array());
-
-    echo "Launching CommitTask daemon...\n";
-    $control->launchDaemon(
-      'PhabricatorRepositoryCommitTaskDaemon',
-      array());
+      $daemon_args);
 
-    echo "NOTE: Make sure you run some taskmaster daemons too, e.g. ".
-         "with 'phd launch 4 taskmaster'.\n";
+    echo "NOTE: '{$command}' is deprecated. Consult the documentation.\n";
 
     echo "Done.\n";
     break;

scripts/install/update_phabricator.sh

@@ -57,9 +57,7 @@ sudo /etc/init.d/httpd start
 # Restart daemons. Customize this to start whatever daemons you're running on
 # your system.
 
-# $ROOT/phabricator/bin/phd repository-launch-master
-# $ROOT/phabricator/bin/phd launch garbagecollector
-# $ROOT/phabricator/bin/phd launch 4 taskmaster
+$ROOT/phabricator/bin/phd start
 # $ROOT/phabricator/bin/phd launch ircbot /config/bot.json
 
 

src/__phutil_library_map__.php

@@ -845,7 +845,6 @@
     'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/base',
     'PhabricatorRepositoryCommitOwnersWorker' => 'applications/repository/worker/owner',
     'PhabricatorRepositoryCommitParserWorker' => 'applications/repository/worker/base',
-    'PhabricatorRepositoryCommitTaskDaemon' => 'applications/repository/daemon/committask',
     'PhabricatorRepositoryController' => 'applications/repository/controller/base',
     'PhabricatorRepositoryCreateController' => 'applications/repository/controller/create',
     'PhabricatorRepositoryDAO' => 'applications/repository/storage/base',
@@ -1734,7 +1733,6 @@
     'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
     'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker',
     'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker',
-    'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorDaemon',
     'PhabricatorRepositoryController' => 'PhabricatorController',
     'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO',

src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php

@@ -269,13 +269,16 @@ private static function recordCommit(
 
     try {
       $commit->save();
+
       $event = new PhabricatorTimelineEvent(
         'cmit',
         array(
           'id' => $commit->getID(),
         ));
       $event->recordEvent();
 
+      self::insertTask($repository, $commit);
+
       queryfx(
         $repository->establishConnection('w'),
         'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
@@ -300,6 +303,35 @@ private static function recordCommit(
     }
   }
 
+  private static function insertTask(
+    PhabricatorRepository $repository,
+    PhabricatorRepositoryCommmit $commit) {
+
+    $vcs = $repository->getVersionControlSystem();
+    switch ($vcs) {
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+        $class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
+        break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+        $class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
+        break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
+        break;
+      default:
+        throw new Exception("Unknown repository type '{$vcs}'!");
+    }
+
+    $task = new PhabricatorWorkerTask();
+    $task->setTaskClass($class);
+    $task->setData(
+      array(
+        'commitID' => $commit->getID(),
+      ));
+    $task->save();
+  }
+
+
   private static function setCache(
     PhabricatorRepository $repository,
     $commit_identifier) {

src/applications/repository/daemon/committask/__init__.php

@@ -15,6 +15,7 @@
 phutil_require_module('phabricator', 'applications/repository/storage/repository');
 phutil_require_module('phabricator', 'infrastructure/daemon/base');
 phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
+phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');
 phutil_require_module('phabricator', 'storage/queryfx');
 
 phutil_require_module('phutil', 'error');