Optimize reindex_everything.php

Summary:
We have two troubles with this script:

# Our revisions and commits don't fit in the memory. (Our tasks do :-).)
# Reindexing revisions is slow.

Test Plan: Ran it.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin

Differential Revision: https://secure.phabricator.com/D3483

Author: vrana

scripts/search/reindex_everything.php

@@ -2,7 +2,7 @@
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,34 +23,28 @@
 // TODO: Get rid of this script eventually, once this stuff is better-formalized
 // in Timeline consumers.
 
-echo "Loading revisions...\n";
-$revs = id(new DifferentialRevision())->loadAll();
-$count = count($revs);
-echo "Reindexing {$count} revisions";
+echo "Reindexing revisions...\n";
+$revs = new LiskMigrationIterator(new DifferentialRevision());
 foreach ($revs as $rev) {
   PhabricatorSearchDifferentialIndexer::indexRevision($rev);
   echo '.';
 }
 echo "\n";
 
-echo "Loading commits...\n";
-$commits = id(new PhabricatorRepositoryCommit())->loadAll();
-$count = count($commits);
-echo "Reindexing {$count} commits";
+echo "Reindexing commits...\n";
+$commits = new LiskMigrationIterator(new PhabricatorRepositoryCommit());
 foreach ($commits as $commit) {
   PhabricatorSearchCommitIndexer::indexCommit($commit);
   echo '.';
 }
 echo "\n";
 
-echo "Loading tasks...\n";
-$tasks = id(new ManiphestTask())->loadAll();
-$count = count($tasks);
-echo "Reindexing {$count} tasks";
+echo "Reindexing tasks...\n";
+$tasks = new LiskMigrationIterator(new ManiphestTask());
 foreach ($tasks as $task) {
   PhabricatorSearchManiphestIndexer::indexTask($task);
   echo '.';
 }
 echo "\n";
-echo "Done.\n";
 
+include dirname(__FILE__).'/reindex_all_users.php';

src/applications/search/index/indexer/PhabricatorSearchDifferentialIndexer.php

@@ -52,13 +52,13 @@ public static function indexRevision(DifferentialRevision $rev) {
         time());
     }
 
-    $comments = id(new DifferentialComment())->loadAllWhere(
-      'revisionID = %d',
-      $rev->getID());
+    $comments = $rev->loadRelatives(new DifferentialComment(), 'revisionID');
 
-    $inlines = id(new DifferentialInlineComment())->loadAllWhere(
-      'revisionID = %d AND commentID IS NOT NULL',
-      $rev->getID());
+    $inlines = $rev->loadRelatives(
+      new DifferentialInlineComment(),
+      'revisionID',
+      'getID',
+      '(commentID IS NOT NULL)');
 
     $touches = array();
 

src/infrastructure/storage/lisk/LiskMigrationIterator.php

@@ -33,9 +33,11 @@ final class LiskMigrationIterator extends PhutilBufferedIterator {
 
   private $object;
   private $cursor;
+  private $set;
 
   public function __construct(LiskDAO $object) {
-    $this->object = $object;
+    $this->set = new LiskDAOSet();
+    $this->object = $object->putInSet($this->set);
   }
 
   protected function didRewind() {
@@ -47,6 +49,8 @@ public function key() {
   }
 
   protected function loadPage() {
+    $this->set->clearSet();
+
     $results = $this->object->loadAllWhere(
       'id > %d ORDER BY id ASC LIMIT %d',
       $this->cursor,