Browse files

Improve Search architecture

Summary:
The search indexing API has several problems right now:

  - Always runs in-process.
    - It would be nice to push this into the task queue for performance. However, the API currently passses an object all the way through (and some indexers depend on preloaded object attributes), so it can't be dumped into the task queue at any stage since we can't serialize it.
    - Being able to use the task queue will also make rebuilding indexes faster.
    - Instead, make the API phid-oriented.
  - No uniform indexing API.
    - Each "Editor" currently calls SomeCustomIndexer::indexThing(). This won't work with AbstractTransactions. The API is also just weird.
    - Instead, provide a uniform API.
  - No uniform CLI.
    - We have `scripts/search/reindex_everything.php`, but it doesn't actually index everything. Each new document type needs to be separately added to it, leading to stuff like D3839. Third-party applications can't provide indexers.
    - Instead, let indexers expose documents for indexing.
  - Not application-oriented.
    - All the indexers live in search/ right now, which isn't the right organization in an application-orietned view of the world.
    - Instead, move indexers to applications and load them with SymbolLoader.

Test Plan:
  - `bin/search index`
    - Indexed one revision, one task.
    - Indexed `--type TASK`, `--type DREV`, etc., for all types.
    - Indexed `--all`.
  - Added the word "saboteur" to a revision, task, wiki page, and question and then searched for it.
    - Creating users is a pain; searched for a user after indexing.
    - Creating commits is a pain; searched for a commit after indexing.
    - Mocks aren't currently loadable in the result view, so their indexing is moot.

Reviewers: btrahan, vrana

Reviewed By: btrahan

CC: 20after4, aran

Maniphest Tasks: T1991, T2104

Differential Revision: https://secure.phabricator.com/D4261
  • Loading branch information...
1 parent aae5f9e commit f6b196474008e3de305db08424be059d946de489 @epriestley epriestley committed Dec 21, 2012
Showing with 386 additions and 161 deletions.
  1. +1 −0 bin/search
  2. +2 −33 scripts/search/index_one_commit.php
  3. +22 −0 scripts/search/manage_search.php
  4. +2 −12 scripts/search/reindex_all_users.php
  5. +2 −31 scripts/search/reindex_everything.php
  6. +20 −15 src/__phutil_library_map__.php
  7. +4 −1 src/applications/audit/editor/PhabricatorAuditCommentEditor.php
  8. +2 −2 src/applications/differential/editor/DifferentialCommentEditor.php
  9. +2 −2 src/applications/differential/editor/DifferentialRevisionEditor.php
  10. +10 −4 ...xer/PhabricatorSearchDifferentialIndexer.php → differential/search/DifferentialSearchIndexer.php}
  11. +2 −1 src/applications/diffusion/controller/DiffusionCommitEditController.php
  12. +2 −3 src/applications/maniphest/editor/ManiphestTransactionEditor.php
  13. +10 −4 ...ndex/indexer/PhabricatorSearchManiphestIndexer.php → maniphest/search/ManiphestSearchIndexer.php}
  14. +9 −6 .../index/indexer/PhabricatorSearchUserIndexer.php → people/search/PhabricatorUserSearchIndexer.php}
  15. +3 −1 src/applications/people/storage/PhabricatorUser.php
  16. +9 −3 src/applications/pholio/indexer/{PholioIndexer.php → PholioSearchIndexer.php}
  17. +3 −1 src/applications/phriction/editor/PhrictionDocumentEditor.php
  18. +13 −4 ...ndex/indexer/PhabricatorSearchPhrictionIndexer.php → phriction/search/PhrictionSearchIndexer.php}
  19. +2 −1 src/applications/ponder/editor/PonderAnswerEditor.php
  20. +2 −2 src/applications/ponder/editor/PonderCommentEditor.php
  21. +2 −2 src/applications/ponder/editor/PonderQuestionEditor.php
  22. +10 −4 src/applications/ponder/search/{PhabricatorSearchPonderIndexer.php → PonderSearchIndexer.php}
  23. +10 −7 ...bricatorSearchCommitIndexer.php → repository/search/PhabricatorRepositoryCommitSearchIndexer.php}
  24. +3 −1 ...plications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
  25. +54 −0 src/applications/search/index/PhabricatorSearchDocumentIndexer.php
  26. +30 −0 src/applications/search/index/PhabricatorSearchIndexer.php
  27. +0 −21 src/applications/search/index/indexer/PhabricatorSearchDocumentIndexer.php
  28. +137 −0 src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
  29. +13 −0 src/applications/search/management/PhabricatorSearchManagementWorkflow.php
  30. +5 −0 src/infrastructure/internationalization/PhabricatorBaseEnglishTranslation.php
View
1 bin/search
View
35 scripts/search/index_one_commit.php
@@ -1,36 +1,5 @@
#!/usr/bin/env php
<?php
-$root = dirname(dirname(dirname(__FILE__)));
-require_once $root.'/scripts/__init_script__.php';
-
-if (empty($argv[1])) {
- echo "usage: index_one_commit.php <commit_name>\n";
- die(1);
-}
-
-$commit = isset($argv[1]) ? $argv[1] : null;
-if (!$commit) {
- throw new Exception("Provide a commit to index!");
-}
-$matches = null;
-if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
- throw new Exception("Can't parse commit identifier!");
-}
-$repo = id(new PhabricatorRepository())->loadOneWhere(
- 'callsign = %s',
- $matches[1]);
-if (!$repo) {
- throw new Exception("Unknown repository!");
-}
-
-$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
- 'repositoryID = %d AND commitIdentifier = %s',
- $repo->getID(),
- $matches[2]);
-if (!$commit) {
- throw new Exception('Unknown commit.');
-}
-
-PhabricatorSearchCommitIndexer::indexCommit($commit);
-echo "Done.\n";
+echo "Use 'bin/search index rXnnnnnn' instead of this script.\n";
+exit(1);
View
22 scripts/search/manage_search.php
@@ -0,0 +1,22 @@
+#!/usr/bin/env php
+<?php
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setTagline('manage search');
+$args->setSynopsis(<<<EOSYNOPSIS
+**search** __command__ [__options__]
+ Manage Phabricator search index.
+
+EOSYNOPSIS
+ );
+$args->parseStandardArguments();
+
+$workflows = array(
+ new PhabricatorSearchManagementIndexWorkflow(),
+ new PhutilHelpArgumentWorkflow(),
+);
+
+$args->parseWorkflows($workflows);
View
14 scripts/search/reindex_all_users.php
@@ -1,15 +1,5 @@
#!/usr/bin/env php
<?php
-$root = dirname(dirname(dirname(__FILE__)));
-require_once $root.'/scripts/__init_script__.php';
-
-$users = id(new PhabricatorUser())->loadAll();
-echo "Indexing ".count($users)." users";
-foreach ($users as $user) {
- PhabricatorSearchUserIndexer::indexUser($user);
- echo '.';
-}
-echo "\n";
-echo "Done.\n";
-
+echo "Use 'bin/search index --type USER' instead of this script.\n";
+die(1);
View
33 scripts/search/reindex_everything.php
@@ -1,34 +1,5 @@
#!/usr/bin/env php
<?php
-$root = dirname(dirname(dirname(__FILE__)));
-require_once $root.'/scripts/__init_script__.php';
-
-// TODO: Get rid of this script eventually, once this stuff is better-formalized
-// in Timeline consumers.
-
-echo "Reindexing revisions...\n";
-$revs = new LiskMigrationIterator(new DifferentialRevision());
-foreach ($revs as $rev) {
- PhabricatorSearchDifferentialIndexer::indexRevision($rev);
- echo '.';
-}
-echo "\n";
-
-echo "Reindexing commits...\n";
-$commits = new LiskMigrationIterator(new PhabricatorRepositoryCommit());
-foreach ($commits as $commit) {
- PhabricatorSearchCommitIndexer::indexCommit($commit);
- echo '.';
-}
-echo "\n";
-
-echo "Reindexing tasks...\n";
-$tasks = new LiskMigrationIterator(new ManiphestTask());
-foreach ($tasks as $task) {
- PhabricatorSearchManiphestIndexer::indexTask($task);
- echo '.';
-}
-echo "\n";
-
-include dirname(__FILE__).'/reindex_all_users.php';
+echo "Use 'bin/search index --all' instead of this script.\n";
+die(1);
View
35 src/__phutil_library_map__.php
@@ -310,6 +310,7 @@
'DifferentialRevisionStatusFieldSpecification' => 'applications/differential/field/specification/DifferentialRevisionStatusFieldSpecification.php',
'DifferentialRevisionUpdateHistoryView' => 'applications/differential/view/DifferentialRevisionUpdateHistoryView.php',
'DifferentialRevisionViewController' => 'applications/differential/controller/DifferentialRevisionViewController.php',
+ 'DifferentialSearchIndexer' => 'applications/differential/search/DifferentialSearchIndexer.php',
'DifferentialSubscribeController' => 'applications/differential/controller/DifferentialSubscribeController.php',
'DifferentialSummaryFieldSpecification' => 'applications/differential/field/specification/DifferentialSummaryFieldSpecification.php',
'DifferentialTasksAttacher' => 'applications/differential/DifferentialTasksAttacher.php',
@@ -533,6 +534,7 @@
'ManiphestSavedQueryDeleteController' => 'applications/maniphest/controller/ManiphestSavedQueryDeleteController.php',
'ManiphestSavedQueryEditController' => 'applications/maniphest/controller/ManiphestSavedQueryEditController.php',
'ManiphestSavedQueryListController' => 'applications/maniphest/controller/ManiphestSavedQueryListController.php',
+ 'ManiphestSearchIndexer' => 'applications/maniphest/search/ManiphestSearchIndexer.php',
'ManiphestSubpriorityController' => 'applications/maniphest/controller/ManiphestSubpriorityController.php',
'ManiphestTask' => 'applications/maniphest/storage/ManiphestTask.php',
'ManiphestTaskAuxiliaryStorage' => 'applications/maniphest/storage/ManiphestTaskAuxiliaryStorage.php',
@@ -1052,6 +1054,7 @@
'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php',
'PhabricatorRepositoryCommitOwnersWorker' => 'applications/repository/worker/PhabricatorRepositoryCommitOwnersWorker.php',
'PhabricatorRepositoryCommitParserWorker' => 'applications/repository/worker/PhabricatorRepositoryCommitParserWorker.php',
+ 'PhabricatorRepositoryCommitSearchIndexer' => 'applications/repository/search/PhabricatorRepositoryCommitSearchIndexer.php',
'PhabricatorRepositoryController' => 'applications/repository/controller/PhabricatorRepositoryController.php',
'PhabricatorRepositoryCreateController' => 'applications/repository/controller/PhabricatorRepositoryCreateController.php',
'PhabricatorRepositoryDAO' => 'applications/repository/storage/PhabricatorRepositoryDAO.php',
@@ -1084,28 +1087,25 @@
'PhabricatorSearchAbstractDocument' => 'applications/search/index/PhabricatorSearchAbstractDocument.php',
'PhabricatorSearchAttachController' => 'applications/search/controller/PhabricatorSearchAttachController.php',
'PhabricatorSearchBaseController' => 'applications/search/controller/PhabricatorSearchBaseController.php',
- 'PhabricatorSearchCommitIndexer' => 'applications/search/index/indexer/PhabricatorSearchCommitIndexer.php',
'PhabricatorSearchController' => 'applications/search/controller/PhabricatorSearchController.php',
'PhabricatorSearchDAO' => 'applications/search/storage/PhabricatorSearchDAO.php',
- 'PhabricatorSearchDifferentialIndexer' => 'applications/search/index/indexer/PhabricatorSearchDifferentialIndexer.php',
'PhabricatorSearchDocument' => 'applications/search/storage/document/PhabricatorSearchDocument.php',
'PhabricatorSearchDocumentField' => 'applications/search/storage/document/PhabricatorSearchDocumentField.php',
- 'PhabricatorSearchDocumentIndexer' => 'applications/search/index/indexer/PhabricatorSearchDocumentIndexer.php',
+ 'PhabricatorSearchDocumentIndexer' => 'applications/search/index/PhabricatorSearchDocumentIndexer.php',
'PhabricatorSearchDocumentRelationship' => 'applications/search/storage/document/PhabricatorSearchDocumentRelationship.php',
'PhabricatorSearchEngine' => 'applications/search/engine/PhabricatorSearchEngine.php',
'PhabricatorSearchEngineElastic' => 'applications/search/engine/PhabricatorSearchEngineElastic.php',
'PhabricatorSearchEngineMySQL' => 'applications/search/engine/PhabricatorSearchEngineMySQL.php',
'PhabricatorSearchEngineSelector' => 'applications/search/selector/PhabricatorSearchEngineSelector.php',
'PhabricatorSearchField' => 'applications/search/constants/PhabricatorSearchField.php',
- 'PhabricatorSearchManiphestIndexer' => 'applications/search/index/indexer/PhabricatorSearchManiphestIndexer.php',
- 'PhabricatorSearchPhrictionIndexer' => 'applications/search/index/indexer/PhabricatorSearchPhrictionIndexer.php',
- 'PhabricatorSearchPonderIndexer' => 'applications/ponder/search/PhabricatorSearchPonderIndexer.php',
+ 'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php',
+ 'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
+ 'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
'PhabricatorSearchQuery' => 'applications/search/storage/PhabricatorSearchQuery.php',
'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php',
'PhabricatorSearchResultView' => 'applications/search/view/PhabricatorSearchResultView.php',
'PhabricatorSearchScope' => 'applications/search/constants/PhabricatorSearchScope.php',
'PhabricatorSearchSelectController' => 'applications/search/controller/PhabricatorSearchSelectController.php',
- 'PhabricatorSearchUserIndexer' => 'applications/search/index/indexer/PhabricatorSearchUserIndexer.php',
'PhabricatorSettingsAdjustController' => 'applications/settings/controller/PhabricatorSettingsAdjustController.php',
'PhabricatorSettingsMainController' => 'applications/settings/controller/PhabricatorSettingsMainController.php',
'PhabricatorSettingsPanel' => 'applications/settings/panel/PhabricatorSettingsPanel.php',
@@ -1191,6 +1191,7 @@
'PhabricatorUserPreferences' => 'applications/settings/storage/PhabricatorUserPreferences.php',
'PhabricatorUserProfile' => 'applications/people/storage/PhabricatorUserProfile.php',
'PhabricatorUserSSHKey' => 'applications/settings/storage/PhabricatorUserSSHKey.php',
+ 'PhabricatorUserSearchIndexer' => 'applications/people/search/PhabricatorUserSearchIndexer.php',
'PhabricatorUserStatus' => 'applications/people/storage/PhabricatorUserStatus.php',
'PhabricatorUserStatusInvalidEpochException' => 'applications/people/exception/PhabricatorUserStatusInvalidEpochException.php',
'PhabricatorUserStatusOverlapException' => 'applications/people/exception/PhabricatorUserStatusOverlapException.php',
@@ -1256,7 +1257,6 @@
'PholioController' => 'applications/pholio/controller/PholioController.php',
'PholioDAO' => 'applications/pholio/storage/PholioDAO.php',
'PholioImage' => 'applications/pholio/storage/PholioImage.php',
- 'PholioIndexer' => 'applications/pholio/indexer/PholioIndexer.php',
'PholioMock' => 'applications/pholio/storage/PholioMock.php',
'PholioMockCommentController' => 'applications/pholio/controller/PholioMockCommentController.php',
'PholioMockEditController' => 'applications/pholio/controller/PholioMockEditController.php',
@@ -1265,6 +1265,7 @@
'PholioMockQuery' => 'applications/pholio/query/PholioMockQuery.php',
'PholioMockViewController' => 'applications/pholio/controller/PholioMockViewController.php',
'PholioReplyHandler' => 'applications/pholio/mail/PholioReplyHandler.php',
+ 'PholioSearchIndexer' => 'applications/pholio/indexer/PholioSearchIndexer.php',
'PholioTransaction' => 'applications/pholio/storage/PholioTransaction.php',
'PholioTransactionComment' => 'applications/pholio/storage/PholioTransactionComment.php',
'PholioTransactionQuery' => 'applications/pholio/query/PholioTransactionQuery.php',
@@ -1291,6 +1292,7 @@
'PhrictionHistoryController' => 'applications/phriction/controller/PhrictionHistoryController.php',
'PhrictionListController' => 'applications/phriction/controller/PhrictionListController.php',
'PhrictionNewController' => 'applications/phriction/controller/PhrictionNewController.php',
+ 'PhrictionSearchIndexer' => 'applications/phriction/search/PhrictionSearchIndexer.php',
'PonderAddAnswerView' => 'applications/ponder/view/PonderAddAnswerView.php',
'PonderAddCommentView' => 'applications/ponder/view/PonderAddCommentView.php',
'PonderAnswer' => 'applications/ponder/storage/PonderAnswer.php',
@@ -1324,6 +1326,7 @@
'PonderQuestionViewController' => 'applications/ponder/controller/PonderQuestionViewController.php',
'PonderReplyHandler' => 'applications/ponder/PonderReplyHandler.php',
'PonderRuleQuestion' => 'infrastructure/markup/rule/PonderRuleQuestion.php',
+ 'PonderSearchIndexer' => 'applications/ponder/search/PonderSearchIndexer.php',
'PonderUserProfileView' => 'applications/ponder/view/PonderUserProfileView.php',
'PonderVotableInterface' => 'applications/ponder/storage/PonderVotableInterface.php',
'PonderVotableView' => 'applications/ponder/view/PonderVotableView.php',
@@ -1631,6 +1634,7 @@
'DifferentialRevisionStatusFieldSpecification' => 'DifferentialFieldSpecification',
'DifferentialRevisionUpdateHistoryView' => 'AphrontView',
'DifferentialRevisionViewController' => 'DifferentialController',
+ 'DifferentialSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
'DifferentialSubscribeController' => 'DifferentialController',
'DifferentialSummaryFieldSpecification' => 'DifferentialFreeformFieldSpecification',
'DifferentialTestPlanFieldSpecification' => 'DifferentialFieldSpecification',
@@ -1815,6 +1819,7 @@
'ManiphestSavedQueryDeleteController' => 'ManiphestController',
'ManiphestSavedQueryEditController' => 'ManiphestController',
'ManiphestSavedQueryListController' => 'ManiphestController',
+ 'ManiphestSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
'ManiphestSubpriorityController' => 'ManiphestController',
'ManiphestTask' =>
array(
@@ -2324,6 +2329,7 @@
'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker',
'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker',
+ 'PhabricatorRepositoryCommitSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
'PhabricatorRepositoryController' => 'PhabricatorController',
'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController',
'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO',
@@ -2351,22 +2357,18 @@
'PhabricatorSSHWorkflow' => 'PhutilArgumentWorkflow',
'PhabricatorSearchAttachController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchBaseController' => 'PhabricatorController',
- 'PhabricatorSearchCommitIndexer' => 'PhabricatorSearchDocumentIndexer',
'PhabricatorSearchController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchDAO' => 'PhabricatorLiskDAO',
- 'PhabricatorSearchDifferentialIndexer' => 'PhabricatorSearchDocumentIndexer',
'PhabricatorSearchDocument' => 'PhabricatorSearchDAO',
'PhabricatorSearchDocumentField' => 'PhabricatorSearchDAO',
'PhabricatorSearchDocumentRelationship' => 'PhabricatorSearchDAO',
'PhabricatorSearchEngineElastic' => 'PhabricatorSearchEngine',
'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine',
- 'PhabricatorSearchManiphestIndexer' => 'PhabricatorSearchDocumentIndexer',
- 'PhabricatorSearchPhrictionIndexer' => 'PhabricatorSearchDocumentIndexer',
- 'PhabricatorSearchPonderIndexer' => 'PhabricatorSearchDocumentIndexer',
+ 'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
+ 'PhabricatorSearchManagementWorkflow' => 'PhutilArgumentWorkflow',
'PhabricatorSearchQuery' => 'PhabricatorSearchDAO',
'PhabricatorSearchResultView' => 'AphrontView',
'PhabricatorSearchSelectController' => 'PhabricatorSearchBaseController',
- 'PhabricatorSearchUserIndexer' => 'PhabricatorSearchDocumentIndexer',
'PhabricatorSettingsAdjustController' => 'PhabricatorController',
'PhabricatorSettingsMainController' => 'PhabricatorController',
'PhabricatorSettingsPanelAccount' => 'PhabricatorSettingsPanel',
@@ -2444,6 +2446,7 @@
'PhabricatorUserPreferences' => 'PhabricatorUserDAO',
'PhabricatorUserProfile' => 'PhabricatorUserDAO',
'PhabricatorUserSSHKey' => 'PhabricatorUserDAO',
+ 'PhabricatorUserSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
'PhabricatorUserStatus' => 'PhabricatorUserDAO',
'PhabricatorUserStatusInvalidEpochException' => 'Exception',
'PhabricatorUserStatusOverlapException' => 'Exception',
@@ -2520,7 +2523,6 @@
0 => 'PholioDAO',
1 => 'PhabricatorMarkupInterface',
),
- 'PholioIndexer' => 'PhabricatorSearchDocumentIndexer',
'PholioMock' =>
array(
0 => 'PholioDAO',
@@ -2535,6 +2537,7 @@
'PholioMockQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
'PholioMockViewController' => 'PholioController',
'PholioReplyHandler' => 'PhabricatorMailReplyHandler',
+ 'PholioSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
'PholioTransaction' => 'PhabricatorApplicationTransaction',
'PholioTransactionComment' => 'PhabricatorApplicationTransactionComment',
'PholioTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
@@ -2564,6 +2567,7 @@
'PhrictionHistoryController' => 'PhrictionController',
'PhrictionListController' => 'PhrictionController',
'PhrictionNewController' => 'PhrictionController',
+ 'PhrictionSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
'PonderAddAnswerView' => 'AphrontView',
'PonderAddCommentView' => 'AphrontView',
'PonderAnswer' =>
@@ -2611,6 +2615,7 @@
'PonderQuestionViewController' => 'PonderController',
'PonderReplyHandler' => 'PhabricatorMailReplyHandler',
'PonderRuleQuestion' => 'PhabricatorRemarkupRuleObjectName',
+ 'PonderSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
'PonderUserProfileView' => 'AphrontView',
'PonderVotableView' => 'AphrontView',
'PonderVoteEditor' => 'PhabricatorEditor',
View
5 src/applications/audit/editor/PhabricatorAuditCommentEditor.php
@@ -289,7 +289,10 @@ public function addComment(PhabricatorAuditComment $comment) {
$feed_phids = array_diff($requests_phids, $feed_dont_publish_phids);
$this->publishFeedStory($comment, $feed_phids);
- PhabricatorSearchCommitIndexer::indexCommit($commit);
+
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($commit->getPHID());
+
$this->sendMail($comment, $other_comments, $inline_comments, $requests);
}
View
4 src/applications/differential/editor/DifferentialCommentEditor.php
@@ -598,8 +598,8 @@ public function save() {
->setMailRecipientPHIDs($mailed_phids)
->publish();
- // TODO: Move to workers
- PhabricatorSearchDifferentialIndexer::indexRevision($revision);
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($revision->getPHID());
return $comment;
}
View
4 src/applications/differential/editor/DifferentialRevisionEditor.php
@@ -489,8 +489,8 @@ public function save() {
->setMailRecipientPHIDs($mailed_phids)
->publish();
- // TODO: Move this into a worker task thing.
- PhabricatorSearchDifferentialIndexer::indexRevision($revision);
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($revision->getPHID());
}
public static function addCCAndUpdateRevision(
View
14 .../PhabricatorSearchDifferentialIndexer.php → ...tial/search/DifferentialSearchIndexer.php
@@ -1,12 +1,18 @@
<?php
/**
- * @group search
+ * @group differential
*/
-final class PhabricatorSearchDifferentialIndexer
+final class DifferentialSearchIndexer
extends PhabricatorSearchDocumentIndexer {
- public static function indexRevision(DifferentialRevision $rev) {
+ public function getIndexableObject() {
+ return new DifferentialRevision();
+ }
+
+ protected function buildAbstractDocumentByPHID($phid) {
+ $rev = $this->loadDocumentByPHID($phid);
+
$doc = new PhabricatorSearchAbstractDocument();
$doc->setPHID($rev->getPHID());
$doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_DREV);
@@ -108,6 +114,6 @@ public static function indexRevision(DifferentialRevision $rev) {
$rev->getDateModified()); // Bogus timestamp.
}
- self::reindexAbstractDocument($doc);
+ return $doc;
}
}
View
3 src/applications/diffusion/controller/DiffusionCommitEditController.php
@@ -44,7 +44,8 @@ public function processRequest() {
}
$editor->save();
- PhabricatorSearchCommitIndexer::indexCommit($commit);
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($commit->getPHID());
return id(new AphrontRedirectResponse())
->setURI('/r'.$callsign.$commit->getCommitIdentifier());
View
5 src/applications/maniphest/editor/ManiphestTransactionEditor.php
@@ -199,9 +199,8 @@ public function applyTransactions(ManiphestTask $task, array $transactions) {
$transactions,
$mail->buildRecipientList());
- // TODO: Do this offline via workers
- PhabricatorSearchManiphestIndexer::indexTask($task);
-
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($task->getPHID());
}
protected function getSubjectPrefix() {
View
14 ...xer/PhabricatorSearchManiphestIndexer.php → ...niphest/search/ManiphestSearchIndexer.php
@@ -1,12 +1,18 @@
<?php
/**
- * @group search
+ * @group maniphest
*/
-final class PhabricatorSearchManiphestIndexer
+final class ManiphestSearchIndexer
extends PhabricatorSearchDocumentIndexer {
- public static function indexTask(ManiphestTask $task) {
+ public function getIndexableObject() {
+ return new ManiphestTask();
+ }
+
+ protected function buildAbstractDocumentByPHID($phid) {
+ $task = $this->loadDocumentByPHID($phid);
+
$doc = new PhabricatorSearchAbstractDocument();
$doc->setPHID($task->getPHID());
$doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_TASK);
@@ -114,6 +120,6 @@ public static function indexTask(ManiphestTask $task) {
$time);
}
- self::reindexAbstractDocument($doc);
+ return $doc;
}
}
View
15 .../indexer/PhabricatorSearchUserIndexer.php → ...e/search/PhabricatorUserSearchIndexer.php
@@ -1,12 +1,15 @@
<?php
-/**
- * @group search
- */
-final class PhabricatorSearchUserIndexer
+final class PhabricatorUserSearchIndexer
extends PhabricatorSearchDocumentIndexer {
- public static function indexUser(PhabricatorUser $user) {
+ public function getIndexableObject() {
+ return new PhabricatorUser();
+ }
+
+ protected function buildAbstractDocumentByPHID($phid) {
+ $user = $this->loadDocumentByPHID($phid);
+
$doc = new PhabricatorSearchAbstractDocument();
$doc->setPHID($user->getPHID());
$doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_USER);
@@ -25,6 +28,6 @@ public static function indexUser(PhabricatorUser $user) {
time());
}
- self::reindexAbstractDocument($doc);
+ return $doc;
}
}
View
4 src/applications/people/storage/PhabricatorUser.php
@@ -104,7 +104,9 @@ public function save() {
$result = parent::save();
$this->updateNameTokens();
- PhabricatorSearchUserIndexer::indexUser($this);
+
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($this->getPHID());
return $result;
}
View
12 ...ications/pholio/indexer/PholioIndexer.php → ...ns/pholio/indexer/PholioSearchIndexer.php
@@ -3,9 +3,15 @@
/**
* @group pholio
*/
-final class PholioIndexer extends PhabricatorSearchDocumentIndexer {
+final class PholioSearchIndexer extends PhabricatorSearchDocumentIndexer {
+
+ public function getIndexableObject() {
+ return new PholioMock();
+ }
+
+ protected function buildAbstractDocumentByPHID($phid) {
+ $mock = $this->loadDocumentByPHID($phid);
- public static function indexMock(PholioMock $mock) {
$doc = new PhabricatorSearchAbstractDocument();
$doc->setPHID($mock->getPHID());
$doc->setDocumentType(phid_get_type($mock->getPHID()));
@@ -23,6 +29,6 @@ public static function indexMock(PholioMock $mock) {
PhabricatorPHIDConstants::PHID_TYPE_USER,
$mock->getDateCreated());
- self::reindexAbstractDocument($doc);
+ return $doc;
}
}
View
4 src/applications/phriction/editor/PhrictionDocumentEditor.php
@@ -172,7 +172,9 @@ private function updateDocument($document, $content, $new_content) {
$document->save();
$document->attachContent($new_content);
- PhabricatorSearchPhrictionIndexer::indexDocument($document);
+
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($document->getPHID());
$project_phid = null;
$slug = $document->getSlug();
View
17 ...xer/PhabricatorSearchPhrictionIndexer.php → ...riction/search/PhrictionSearchIndexer.php
@@ -1,12 +1,21 @@
<?php
/**
- * @group search
+ * @group phriction
*/
-final class PhabricatorSearchPhrictionIndexer
+final class PhrictionSearchIndexer
extends PhabricatorSearchDocumentIndexer {
- public static function indexDocument(PhrictionDocument $document) {
+ public function getIndexableObject() {
+ return new PhrictionDocument();
+ }
+
+ protected function buildAbstractDocumentByPHID($phid) {
+ $document = $this->loadDocumentByPHID($phid);
+
+ $content = id(new PhrictionContent())->load($document->getContentID());
+ $document->attachContent($content);
+
$content = $document->getContent();
$doc = new PhabricatorSearchAbstractDocument();
@@ -28,6 +37,6 @@ public static function indexDocument(PhrictionDocument $document) {
PhabricatorPHIDConstants::PHID_TYPE_USER,
$content->getDateCreated());
- self::reindexAbstractDocument($doc);
+ return $doc;
}
}
View
3 src/applications/ponder/editor/PonderAnswerEditor.php
@@ -47,7 +47,8 @@ public function saveAnswer() {
$trans->saveTransaction();
$question->attachRelated();
- PhabricatorSearchPonderIndexer::indexQuestion($question);
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($question->getPHID());
// subscribe author and @mentions
$subeditor = id(new PhabricatorSubscriptionsEditor())
View
4 src/applications/ponder/editor/PonderCommentEditor.php
@@ -39,8 +39,8 @@ public function save() {
$target = $this->targetPHID;
$comment->save();
- $question->attachRelated();
- PhabricatorSearchPonderIndexer::indexQuestion($question);
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($question->getPHID());
// subscribe author and @mentions
$subeditor = id(new PhabricatorSubscriptionsEditor())
View
4 src/applications/ponder/editor/PonderQuestionEditor.php
@@ -24,9 +24,9 @@ public function save() {
$question = $this->question;
$question->save();
- // search index
$question->attachRelated();
- PhabricatorSearchPonderIndexer::indexQuestion($question);
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($question->getPHID());
// subscribe author and @mentions
$subeditor = id(new PhabricatorSubscriptionsEditor())
View
14 ...search/PhabricatorSearchPonderIndexer.php → ...ons/ponder/search/PonderSearchIndexer.php
@@ -1,10 +1,16 @@
<?php
-final class PhabricatorSearchPonderIndexer
+final class PonderSearchIndexer
extends PhabricatorSearchDocumentIndexer {
- public static function indexQuestion(PonderQuestion $question) {
- // note: we assume someone's already called attachrelated on $question
+ public function getIndexableObject() {
+ return new PonderQuestion();
+ }
+
+ protected function buildAbstractDocumentByPHID($phid) {
+ $question = $this->loadDocumentByPHID($phid);
+
+ $question->attachRelated();
$doc = new PhabricatorSearchAbstractDocument();
$doc->setPHID($question->getPHID());
@@ -61,6 +67,6 @@ public static function indexQuestion(PonderQuestion $question) {
$question->getDateModified()); // Bogus timestamp.
}
- self::reindexAbstractDocument($doc);
+ return $doc;
}
}
View
17 ...ndexer/PhabricatorSearchCommitIndexer.php → ...bricatorRepositoryCommitSearchIndexer.php
@@ -1,12 +1,15 @@
<?php
-/**
- * @group search
- */
-final class PhabricatorSearchCommitIndexer
+final class PhabricatorRepositoryCommitSearchIndexer
extends PhabricatorSearchDocumentIndexer {
- public static function indexCommit(PhabricatorRepositoryCommit $commit) {
+ public function getIndexableObject() {
+ return new PhabricatorRepositoryCommit();
+ }
+
+ protected function buildAbstractDocumentByPHID($phid) {
+ $commit = $this->loadDocumentByPHID($phid);
+
$commit_data = id(new PhabricatorRepositoryCommitData())->loadOneWhere(
'commitID = %d',
$commit->getID());
@@ -19,7 +22,7 @@ public static function indexCommit(PhabricatorRepositoryCommit $commit) {
$commit->getRepositoryID());
if (!$repository) {
- return;
+ throw new Exception("No such repository!");
}
$title = 'r'.$repository->getCallsign().$commit->getCommitIdentifier().
@@ -75,7 +78,7 @@ public static function indexCommit(PhabricatorRepositoryCommit $commit) {
}
}
- self::reindexAbstractDocument($doc);
+ return $doc;
}
}
View
4 ...ns/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
@@ -57,7 +57,9 @@ private static function lookupPaths(array $paths) {
protected function finishParse() {
$commit = $this->commit;
- PhabricatorSearchCommitIndexer::indexCommit($commit);
+
+ id(new PhabricatorSearchIndexer())
+ ->indexDocumentByPHID($commit->getPHID());
PhabricatorOwnersPackagePathValidator::updateOwnersPackagePaths($commit);
if ($this->shouldQueueFollowupTasks()) {
View
54 src/applications/search/index/PhabricatorSearchDocumentIndexer.php
@@ -0,0 +1,54 @@
+<?php
+
+/**
+ * @group search
+ */
+abstract class PhabricatorSearchDocumentIndexer {
+
+ abstract public function getIndexableObject();
+ abstract protected function buildAbstractDocumentByPHID($phid);
+
+ public function shouldIndexDocumentByPHID($phid) {
+ $object = $this->getIndexableObject();
+ return (phid_get_type($phid) == phid_get_type($object->generatePHID()));
+ }
+
+ public function getIndexIterator() {
+ $object = $this->getIndexableObject();
+ return new LiskMigrationIterator($object);
+ }
+
+ protected function loadDocumentByPHID($phid) {
+ $object = $this->getIndexableObject();
+ $document = $object->loadOneWhere('phid = %s', $phid);
+ if (!$document) {
+ throw new Exception("Unable to load document by phid '{$phid}'!");
+ }
+ return $document;
+ }
+
+ public function indexDocumentByPHID($phid) {
+ try {
+ $document = $this->buildAbstractDocumentByPHID($phid);
+
+ $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
+ try {
+ $engine->reindexAbstractDocument($document);
+ } catch (Exception $ex) {
+ $phid = $document->getPHID();
+ $class = get_class($engine);
+
+ phlog("Unable to index document {$phid} by engine {$class}.");
+ phlog($ex);
+ }
+
+ } catch (Exception $ex) {
+ $class = get_class($this);
+ phlog("Unable to build document {$phid} by indexer {$class}.");
+ phlog($ex);
+ }
+
+ return $this;
+ }
+
+}
View
30 src/applications/search/index/PhabricatorSearchIndexer.php
@@ -0,0 +1,30 @@
+<?php
+
+/**
+ * @group search
+ */
+final class PhabricatorSearchIndexer {
+
+ public function indexDocumentByPHID($phid) {
+ $doc_indexer_symbols = id(new PhutilSymbolLoader())
+ ->setAncestorClass('PhabricatorSearchDocumentIndexer')
+ ->setConcreteOnly(true)
+ ->setType('class')
+ ->selectAndLoadSymbols();
+
+ $indexers = array();
+ foreach ($doc_indexer_symbols as $symbol) {
+ $indexers[] = newv($symbol['name'], array());
+ }
+
+ foreach ($indexers as $indexer) {
+ if ($indexer->shouldIndexDocumentByPHID($phid)) {
+ $indexer->indexDocumentByPHID($phid);
+ break;
+ }
+ }
+
+ return $this;
+ }
+
+}
View
21 src/applications/search/index/indexer/PhabricatorSearchDocumentIndexer.php
@@ -1,21 +0,0 @@
-<?php
-
-/**
- * @group search
- */
-abstract class PhabricatorSearchDocumentIndexer {
-
- // TODO: Make this whole class tree concrete?
- final protected static function reindexAbstractDocument(
- PhabricatorSearchAbstractDocument $document) {
- $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
- try {
- $engine->reindexAbstractDocument($document);
- } catch (Exception $ex) {
- $phid = $document->getPHID();
- $class = get_class($engine);
- phlog("Unable to index document {$phid} by engine {$class}.");
- }
- }
-
-}
View
137 src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
@@ -0,0 +1,137 @@
+<?php
+
+/**
+ * @group search
+ */
+final class PhabricatorSearchManagementIndexWorkflow
+ extends PhabricatorSearchManagementWorkflow {
+
+ protected function didConstruct() {
+ $this
+ ->setName('index')
+ ->setSynopsis('Build or rebuild search indexes.')
+ ->setExamples(
+ "**index** D123\n".
+ "**index** --type DREV\n".
+ "**index** --all")
+ ->setArguments(
+ array(
+ array(
+ 'name' => 'all',
+ 'help' => 'Reindex all documents.',
+ ),
+ array(
+ 'name' => 'type',
+ 'param' => 'TYPE',
+ 'help' => 'PHID type to reindex, like "TASK" or "DREV".',
+ ),
+ array(
+ 'name' => 'background',
+ 'help' => 'Instead of indexing in this process, queue tasks for '.
+ 'the daemons. This is better if you are indexing a lot '.
+ 'of stuff, but less helpful for debugging.',
+ ),
+ array(
+ 'name' => 'foreground',
+ 'help' => 'Index in this process, even if there are many objects '.
+ 'to index. This is helpful for debugging.',
+ ),
+ array(
+ 'name' => 'objects',
+ 'wildcard' => true,
+ ),
+ )
+ );
+ }
+
+ public function execute(PhutilArgumentParser $args) {
+ $console = PhutilConsole::getConsole();
+
+ $is_all = $args->getArg('all');
+ $is_type = $args->getArg('type');
+
+ $obj_names = $args->getArg('objects');
+
+
+ if ($obj_names && ($is_all || $is_type)) {
+ throw new PhutilArgumentUsageException(
+ "You can not name objects to index alongside the '--all' or '--type' ".
+ "flags.");
+ } else if (!$obj_names && !($is_all || $is_type)) {
+ throw new PhutilArgumentUsageException(
+ "Provide one of '--all', '--type' or a list of object names.");
+ }
+
+ if ($obj_names) {
+ $phids = $this->loadPHIDsByNames($obj_names);
+ } else {
+ $phids = $this->loadPHIDsByTypes($is_type);
+ }
+
+ if (!$phids) {
+ throw new PhutilArgumentUsageException(
+ "Nothing to index!");
+ }
+
+ $groups = phid_group_by_type($phids);
+ foreach ($groups as $group_type => $group) {
+ $console->writeOut(
+ pht(
+ "Indexing %d object(s) of type %s.",
+ count($group),
+ $group_type)."\n");
+ }
+
+ $indexer = new PhabricatorSearchIndexer();
+ foreach ($phids as $phid) {
+ $indexer->indexDocumentByPHID($phid);
+ $console->writeOut(pht("Indexing '%s'...\n", $phid));
+ }
+
+ $console->writeOut("Done.\n");
+ }
+
+ private function loadPHIDsByNames(array $names) {
+ $phids = array();
+ foreach ($names as $name) {
+ $phid = PhabricatorPHID::fromObjectName($name);
+ if (!$phid) {
+ throw new PhutilArgumentUsageException(
+ "'{$name}' is not the name of a known object.");
+ }
+ $phids[] = $phid;
+ }
+ return $phids;
+ }
+
+ private function loadPHIDsByTypes($type) {
+ $indexer_symbols = id(new PhutilSymbolLoader())
+ ->setAncestorClass('PhabricatorSearchDocumentIndexer')
+ ->setConcreteOnly(true)
+ ->setType('class')
+ ->selectAndLoadSymbols();
+
+ $indexers = array();
+ foreach ($indexer_symbols as $symbol) {
+ $indexers[] = newv($symbol['name'], array());
+ }
+
+ $phids = array();
+ foreach ($indexers as $indexer) {
+ $indexer_phid = $indexer->getIndexableObject()->generatePHID();
+ $indexer_type = phid_get_type($indexer_phid);
+
+ if ($type && ($indexer_type != $type)) {
+ continue;
+ }
+
+ $iterator = $indexer->getIndexIterator();
+ foreach ($iterator as $object) {
+ $phids[] = $object->getPHID();
+ }
+ }
+
+ return $phids;
+ }
+
+}
View
13 src/applications/search/management/PhabricatorSearchManagementWorkflow.php
@@ -0,0 +1,13 @@
+<?php
+
+/**
+ * @group search
+ */
+abstract class PhabricatorSearchManagementWorkflow
+ extends PhutilArgumentWorkflow {
+
+ final public function isExecutable() {
+ return true;
+ }
+
+}
View
5 src/infrastructure/internationalization/PhabricatorBaseEnglishTranslation.php
@@ -198,6 +198,11 @@ public function getTranslations() {
'%s Lines',
),
+ "Indexing %d object(s) of type %s." => array(
+ "Indexing %d object of type %s.",
+ "Indexing %d object of type %s.",
+ ),
+
);
}

0 comments on commit f6b1964

Please sign in to comment.