Move 'phd parse-commit' to a dedicated script; allow message parsing tasks to

be executed in isolation, provide a script to requeue all message reparses,
stop parse-commit from inserting side-effect tasks.

Author: epriestley

scripts/daemon/phabricator_daemon_launcher.php

@@ -169,59 +169,6 @@
 
     break;
 
-  case 'parse-commit':
-    $commit = isset($argv[2]) ? $argv[2] : null;
-    if (!$commit) {
-      throw new Exception("Provide a commit to parse!");
-    }
-    $matches = null;
-    if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
-      throw new Exception("Can't parse commit identifier!");
-    }
-    $repo = id(new PhabricatorRepository())->loadOneWhere(
-      'callsign = %s',
-      $matches[1]);
-    if (!$repo) {
-      throw new Exception("Unknown repository!");
-    }
-    $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
-      'repositoryID = %d AND commitIdentifier = %s',
-      $repo->getID(),
-      $matches[2]);
-    if (!$commit) {
-      throw new Exception('Unknown commit.');
-    }
-
-    $workers = array();
-
-
-    switch ($repo->getVersionControlSystem()) {
-      case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
-        $workers[] = new PhabricatorRepositoryGitCommitMessageParserWorker(
-          $commit->getID());
-        $workers[] = new PhabricatorRepositoryGitCommitChangeParserWorker(
-          $commit->getID());
-        break;
-      case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
-        $workers[] = new PhabricatorRepositorySvnCommitMessageParserWorker(
-          $commit->getID());
-        $workers[] = new PhabricatorRepositorySvnCommitChangeParserWorker(
-          $commit->getID());
-        break;
-      default:
-        throw new Exception("Unknown repository type!");
-    }
-
-    ExecFuture::pushEchoMode(true);
-
-    foreach ($workers as $worker) {
-      echo "Running ".get_class($worker)."...\n";
-      $worker->doWork();
-    }
-
-    echo "Done.\n";
-
-    break;
   case '--help':
   case 'help':
   default:

scripts/repository/parse_one_commit.php

@@ -0,0 +1,83 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+require_once $root.'/scripts/__init_env__.php';
+
+if (empty($argv[1])) {
+  echo "usage: parse_one_commit.php <commit_name>\n";
+  die(1);
+}
+
+$commit = isset($argv[1]) ? $argv[1] : null;
+if (!$commit) {
+  throw new Exception("Provide a commit to parse!");
+}
+$matches = null;
+if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
+  throw new Exception("Can't parse commit identifier!");
+}
+$repo = id(new PhabricatorRepository())->loadOneWhere(
+  'callsign = %s',
+  $matches[1]);
+if (!$repo) {
+  throw new Exception("Unknown repository!");
+}
+$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
+  'repositoryID = %d AND commitIdentifier = %s',
+  $repo->getID(),
+  $matches[2]);
+if (!$commit) {
+  throw new Exception('Unknown commit.');
+}
+
+$workers = array();
+
+$spec = array(
+  'commitID'  => $commit->getID(),
+  'only'      => true,
+);
+
+switch ($repo->getVersionControlSystem()) {
+  case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+    $workers[] = new PhabricatorRepositoryGitCommitMessageParserWorker(
+      $spec);
+    $workers[] = new PhabricatorRepositoryGitCommitChangeParserWorker(
+      $spec);
+    break;
+  case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+    $workers[] = new PhabricatorRepositorySvnCommitMessageParserWorker(
+      $spec);
+    $workers[] = new PhabricatorRepositorySvnCommitChangeParserWorker(
+      $spec);
+    break;
+  default:
+    throw new Exception("Unknown repository type!");
+}
+
+ExecFuture::pushEchoMode(true);
+
+foreach ($workers as $worker) {
+  echo "Running ".get_class($worker)."...\n";
+  $worker->doWork();
+}
+
+echo "Done.\n";
+

scripts/repository/reparse_all_commit_messages.php

@@ -0,0 +1,95 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+require_once $root.'/scripts/__init_env__.php';
+
+phutil_require_module('phutil', 'console');
+
+if (empty($argv[1])) {
+  echo "usage: reparse_all_commit_messages.php all\n".
+       "       reparse_all_commit_messages.php <repository_callsign>\n";
+  exit(1);
+}
+
+
+echo phutil_console_format(
+  'This script will queue tasks to reparse every commit message known to '.
+  'Diffusion. Once the tasks have been inserted, you need to start '.
+  'Taskmaster daemons to execute them.');
+
+$ok = phutil_console_confirm('Do you want to continue?');
+if (!$ok) {
+  die(1);
+}
+
+if ($argv[1] == 'all') {
+  echo "Loading all repositories...\n";
+  $repositories = id(new PhabricatorRepository())->loadAll();
+  echo "Loading all commits...\n";
+  $commits = id(new PhabricatorRepositoryCommit())->loadAll();
+} else {
+  $callsign = $argv[1];
+  echo "Loading '{$callsign}' repository...\n";
+  $repository = id(new PhabricatorRepository())->loadOneWhere(
+    'callsign = %s',
+    $argv[1]);
+  if (!$repository) {
+    throw new Exception("No such repository exists!");
+  }
+  $repositories = array(
+    $repository->getID() => $repository,
+  );
+  echo "Loading commits in '{$callsign}' repository...\n";
+  $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
+    'repositoryID = %d',
+    $repository->getID());
+}
+
+echo "Inserting tasks for ".count($commits)." commits";
+foreach ($commits as $commit) {
+  echo ".";
+  $id = $commit->getID();
+  $repo = idx($repositories, $commit->getRepositoryID());
+  if (!$repo) {
+    echo "\nWarning: Commit #{$id} has an invalid repository ID.\n";
+  }
+
+  switch ($repo->getVersionControlSystem()) {
+    case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+      $task_class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
+      break;
+    case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+      $task_class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
+      break;
+    default:
+      throw new Exception("Unknown repository type!");
+  }
+
+  $task = new PhabricatorWorkerTask();
+  $task->setTaskClass($task_class);
+  $task->setData(
+    array(
+      'commitID'  => $commit->getID(),
+      'only'      => true,
+    ));
+  $task->save();
+}
+echo "\nDone.\n";

src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php

@@ -47,14 +47,20 @@ final public function run() {
             $class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
             $task = new PhabricatorWorkerTask();
             $task->setTaskClass($class);
-            $task->setData($commit->getID());
+            $task->setData(
+              array(
+                'commitID' => $commit->getID(),
+              ));
             $task->save();
             break;
           case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
             $class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
             $task = new PhabricatorWorkerTask();
             $task->setTaskClass($class);
-            $task->setData($commit->getID());
+            $task->setData(
+              array(
+                'commitID' => $commit->getID(),
+              ));
             $task->save();
             break;
           default:

src/applications/repository/worker/base/PhabricatorRepositoryCommitParserWorker.php

@@ -23,7 +23,7 @@ abstract class PhabricatorRepositoryCommitParserWorker
   protected $repository;
 
   final public function doWork() {
-    $commit_id = $this->getTaskData();
+    $commit_id = idx($this->getTaskData(), 'commitID');
     if (!$commit_id) {
       return;
     }
@@ -49,6 +49,10 @@ final public function doWork() {
     return $this->parseCommit($repository, $commit);
   }
 
+  final protected function shouldQueueFollowupTasks() {
+    return !!idx($this->getTaskData(), 'only');
+  }
+
   abstract protected function parseCommit(
     PhabricatorRepository $repository,
     PhabricatorRepositoryCommit $commit);

src/applications/repository/worker/commitmessageparser/git/PhabricatorRepositoryGitCommitMessageParserWorker.php

@@ -30,16 +30,24 @@ public function parseCommit(
       $local_path,
       $commit->getCommitIdentifier());
 
-    // TODO: Need to slam UTF8?
 
     list($author, $message) = explode("\0", $info);
 
+    // Make sure these are valid UTF-8.
+    $author = phutil_utf8ize($author);
+    $message = phutil_utf8ize($message);
+
     $this->updateCommitData($author, $message);
 
-    $task = new PhabricatorWorkerTask();
-    $task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker');
-    $task->setData($commit->getID());
-    $task->save();
+    if ($this->shouldQueueFollowupTasks()) {
+      $task = new PhabricatorWorkerTask();
+      $task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker');
+      $task->setData(
+        array(
+          'commitID' => $commit->getID(),
+        ));
+      $task->save();
+    }
   }
 
 }

src/applications/repository/worker/commitmessageparser/svn/PhabricatorRepositorySvnCommitMessageParserWorker.php

@@ -37,10 +37,15 @@ public function parseCommit(
 
     $this->updateCommitData($author, $message);
 
-    $task = new PhabricatorWorkerTask();
-    $task->setTaskClass('PhabricatorRepositorySvnCommitChangeParserWorker');
-    $task->setData($commit->getID());
-    $task->save();
+    if ($this->shouldQueueFollowupTasks()) {
+      $task = new PhabricatorWorkerTask();
+      $task->setTaskClass('PhabricatorRepositorySvnCommitChangeParserWorker');
+      $task->setData(
+        array(
+          'commitID' => $commit->getID(),
+        ));
+      $task->save();
+    }
   }
 
 }

src/infrastructure/daemon/control/PhabricatorDaemonControl.php

@@ -145,9 +145,6 @@ public function executeHelpCommand() {
         **help**
             Show this help.
 
-        **parse-commit** __rXnnnn__
-            Parse a single commit.
-
         **repository-launch-master**
             Launches daemons to update and parse all tracked repositories. You
             must also launch Taskmaster daemons, either on the same machine or