Skip to content

Commit 99bd12b

Browse files
author
epriestley
committedDec 22, 2015
Lift Conpherence indexing up out of the Fulltext index
Summary: Ref T9979. There are currently some hacks around Conpherence indexing: it does not really use the fulltext index, but its own specialized index. However, it's kind of hacked up so it can get reindexed by the normal indexing pipeline. Lift it up into IndexEngine, instead of FulltextEngine. Specifically, the new stuff is going to look like this: - IndexEngine: Rebuild all indexes. - ConpherenceIndexExtension: Rebuild thread indexes. - ProjectMemberIndexExtension: Rebuild project membership views. - NgramIndexExtension: Rebuild ngram indexes. - FulltextIndexExtension / FulltextEngine: Rebuild fulltext indexes, a special type of index. - FulltextCommentExtension: Rebuild comment fulltext indexes. - FulltextProjectExtension: Rebuild project fulltext indexes. - etc. Most of this is at least sort-of-in-place as of this diff, although some of the part in the middle is still pretty rough. Test Plan: - Made a unique comment in a Conpherence thread. - Used `bin/search index --force` to rebuild the index. - Searched for the comment. - Found the thread. Reviewers: chad Reviewed By: chad Maniphest Tasks: T9979 Differential Revision: https://secure.phabricator.com/D14841
1 parent ecc3314 commit 99bd12b

10 files changed

+260
-84
lines changed
 

‎src/__phutil_library_map__.php

+6-2
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@
285285
'ConpherenceSettings' => 'applications/conpherence/constants/ConpherenceSettings.php',
286286
'ConpherenceTestCase' => 'applications/conpherence/__tests__/ConpherenceTestCase.php',
287287
'ConpherenceThread' => 'applications/conpherence/storage/ConpherenceThread.php',
288-
'ConpherenceThreadIndexer' => 'applications/conpherence/search/ConpherenceThreadIndexer.php',
288+
'ConpherenceThreadIndexEngineExtension' => 'applications/conpherence/engineextension/ConpherenceThreadIndexEngineExtension.php',
289289
'ConpherenceThreadListView' => 'applications/conpherence/view/ConpherenceThreadListView.php',
290290
'ConpherenceThreadMailReceiver' => 'applications/conpherence/mail/ConpherenceThreadMailReceiver.php',
291291
'ConpherenceThreadMembersPolicyRule' => 'applications/conpherence/policyrule/ConpherenceThreadMembersPolicyRule.php',
@@ -2378,6 +2378,8 @@
23782378
'PhabricatorImageTransformer' => 'applications/files/PhabricatorImageTransformer.php',
23792379
'PhabricatorImagemagickSetupCheck' => 'applications/config/check/PhabricatorImagemagickSetupCheck.php',
23802380
'PhabricatorIndexEngine' => 'applications/search/index/PhabricatorIndexEngine.php',
2381+
'PhabricatorIndexEngineExtension' => 'applications/search/index/PhabricatorIndexEngineExtension.php',
2382+
'PhabricatorIndexEngineExtensionModule' => 'applications/search/index/PhabricatorIndexEngineExtensionModule.php',
23812383
'PhabricatorInfrastructureTestCase' => '__tests__/PhabricatorInfrastructureTestCase.php',
23822384
'PhabricatorInlineCommentController' => 'infrastructure/diff/PhabricatorInlineCommentController.php',
23832385
'PhabricatorInlineCommentInterface' => 'infrastructure/diff/interface/PhabricatorInlineCommentInterface.php',
@@ -4181,7 +4183,7 @@
41814183
'PhabricatorMentionableInterface',
41824184
'PhabricatorDestructibleInterface',
41834185
),
4184-
'ConpherenceThreadIndexer' => 'PhabricatorSearchDocumentIndexer',
4186+
'ConpherenceThreadIndexEngineExtension' => 'PhabricatorIndexEngineExtension',
41854187
'ConpherenceThreadListView' => 'AphrontView',
41864188
'ConpherenceThreadMailReceiver' => 'PhabricatorObjectMailReceiver',
41874189
'ConpherenceThreadMembersPolicyRule' => 'PhabricatorPolicyRule',
@@ -6606,6 +6608,8 @@
66066608
'PhabricatorImageTransformer' => 'Phobject',
66076609
'PhabricatorImagemagickSetupCheck' => 'PhabricatorSetupCheck',
66086610
'PhabricatorIndexEngine' => 'Phobject',
6611+
'PhabricatorIndexEngineExtension' => 'Phobject',
6612+
'PhabricatorIndexEngineExtensionModule' => 'PhabricatorConfigModule',
66096613
'PhabricatorInfrastructureTestCase' => 'PhabricatorTestCase',
66106614
'PhabricatorInlineCommentController' => 'PhabricatorController',
66116615
'PhabricatorInlineCommentInterface' => 'PhabricatorMarkupInterface',

‎src/applications/conpherence/editor/ConpherenceEditor.php

-16
Original file line numberDiff line numberDiff line change
@@ -601,22 +601,6 @@ protected function supportsSearch() {
601601
return true;
602602
}
603603

604-
protected function getSearchContextParameter(
605-
PhabricatorLiskDAO $object,
606-
array $xactions) {
607-
608-
$comment_phids = array();
609-
foreach ($xactions as $xaction) {
610-
if ($xaction->hasComment()) {
611-
$comment_phids[] = $xaction->getPHID();
612-
}
613-
}
614-
615-
return array(
616-
'commentPHIDs' => $comment_phids,
617-
);
618-
}
619-
620604
protected function extractFilePHIDsFromCustomTransaction(
621605
PhabricatorLiskDAO $object,
622606
PhabricatorApplicationTransaction $xaction) {

‎src/applications/conpherence/search/ConpherenceThreadIndexer.php ‎src/applications/conpherence/engineextension/ConpherenceThreadIndexEngineExtension.php

+25-33
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,50 @@
11
<?php
22

3-
final class ConpherenceThreadIndexer
4-
extends PhabricatorSearchDocumentIndexer {
3+
final class ConpherenceThreadIndexEngineExtension
4+
extends PhabricatorIndexEngineExtension {
55

6-
public function getIndexableObject() {
7-
return new ConpherenceThread();
8-
}
9-
10-
protected function loadDocumentByPHID($phid) {
11-
$object = id(new ConpherenceThreadQuery())
12-
->setViewer($this->getViewer())
13-
->withPHIDs(array($phid))
14-
->executeOne();
15-
16-
if (!$object) {
17-
throw new Exception(pht('No thread "%s" exists!', $phid));
18-
}
6+
const EXTENSIONKEY = 'conpherence.thread';
197

20-
return $object;
8+
public function getExtensionName() {
9+
return pht('Conpherence Threads');
2110
}
2211

23-
protected function buildAbstractDocumentByPHID($phid) {
24-
$thread = $this->loadDocumentByPHID($phid);
12+
public function shouldIndexObject($object) {
13+
return ($object instanceof ConpherenceThread);
14+
}
2515

26-
// NOTE: We're explicitly not building a document here, only rebuilding
27-
// the Conpherence search index.
16+
public function indexObject(
17+
PhabricatorIndexEngine $engine,
18+
$object) {
2819

29-
$context = nonempty($this->getContext(), array());
30-
$comment_phids = idx($context, 'commentPHIDs');
20+
$force = $this->shouldForceFullReindex();
3121

32-
if (is_array($comment_phids) && !$comment_phids) {
33-
// If this property is set, but empty, the transaction did not
34-
// include any chat text. For example, a user might have left the
35-
// conversation.
36-
return null;
22+
if (!$force) {
23+
$xaction_phids = $this->getParameter('transactionPHIDs');
24+
if (!$xaction_phids) {
25+
return;
26+
}
3727
}
3828

3929
$query = id(new ConpherenceTransactionQuery())
4030
->setViewer($this->getViewer())
41-
->withObjectPHIDs(array($thread->getPHID()))
31+
->withObjectPHIDs(array($object->getPHID()))
4232
->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT))
4333
->needComments(true);
4434

45-
if ($comment_phids !== null) {
46-
$query->withPHIDs($comment_phids);
35+
if (!$force) {
36+
$query->withPHIDs($xaction_phids);
4737
}
4838

4939
$xactions = $query->execute();
5040

51-
foreach ($xactions as $xaction) {
52-
$this->indexComment($thread, $xaction);
41+
if (!$xactions) {
42+
return;
5343
}
5444

55-
return null;
45+
foreach ($xactions as $xaction) {
46+
$this->indexComment($object, $xaction);
47+
}
5648
}
5749

5850
private function indexComment(

‎src/applications/search/index/PhabricatorIndexEngine.php

+96-2
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,108 @@
22

33
final class PhabricatorIndexEngine extends Phobject {
44

5-
public function indexDocumentByPHID($phid, $context) {
5+
private $object;
6+
private $extensions;
7+
private $versions;
8+
private $parameters;
9+
10+
public function setParameters(array $parameters) {
11+
$this->parameters = $parameters;
12+
return $this;
13+
}
14+
15+
public function getParameters() {
16+
return $this->parameters;
17+
}
18+
19+
public function setObject($object) {
20+
$this->object = $object;
21+
return $this;
22+
}
23+
24+
public function getObject() {
25+
return $this->object;
26+
}
27+
28+
public function shouldIndexObject() {
29+
$extensions = $this->newExtensions();
30+
31+
$parameters = $this->getParameters();
32+
foreach ($extensions as $extension) {
33+
$extension->setParameters($parameters);
34+
}
35+
36+
$object = $this->getObject();
37+
$versions = array();
38+
foreach ($extensions as $key => $extension) {
39+
$version = $extension->getIndexVersion($object);
40+
if ($version !== null) {
41+
$versions[$key] = (string)$version;
42+
}
43+
}
44+
45+
if (idx($parameters, 'force')) {
46+
$current_versions = array();
47+
} else {
48+
// TODO: Load current indexed versions.
49+
$current_versions = array();
50+
}
51+
52+
foreach ($versions as $key => $version) {
53+
$current_version = idx($current_versions, $key);
54+
55+
if ($current_version === null) {
56+
continue;
57+
}
58+
59+
// If nothing has changed since we built the current index, we do not
60+
// need to rebuild the index.
61+
if ($current_version === $version) {
62+
unset($extensions[$key]);
63+
}
64+
}
65+
66+
$this->extensions = $extensions;
67+
$this->versions = $versions;
68+
69+
// We should index the object only if there is any work to be done.
70+
return (bool)$this->extensions;
71+
}
72+
73+
public function indexObject() {
74+
$extensions = $this->extensions;
75+
$object = $this->getObject();
76+
77+
foreach ($extensions as $key => $extension) {
78+
$extension->indexObject($this, $object);
79+
}
80+
81+
// TODO: Save new index versions.
82+
83+
return $this;
84+
}
85+
86+
private function newExtensions() {
87+
$object = $this->getObject();
88+
89+
$extensions = PhabricatorIndexEngineExtension::getAllExtensions();
90+
foreach ($extensions as $key => $extension) {
91+
if (!$extension->shouldIndexObject($object)) {
92+
unset($extensions[$key]);
93+
}
94+
}
95+
96+
return $extensions;
97+
}
98+
99+
public function indexDocumentByPHID($phid) {
6100
$indexers = id(new PhutilClassMapQuery())
7101
->setAncestorClass('PhabricatorSearchDocumentIndexer')
8102
->execute();
9103

10104
foreach ($indexers as $indexer) {
11105
if ($indexer->shouldIndexDocumentByPHID($phid)) {
12-
$indexer->indexDocumentByPHID($phid, $context);
106+
$indexer->indexDocumentByPHID($phid);
13107
break;
14108
}
15109
}

‎src/applications/search/index/PhabricatorIndexEngineExtension.php

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
<?php
2+
3+
abstract class PhabricatorIndexEngineExtension extends Phobject {
4+
5+
private $parameters;
6+
private $forceFullReindex;
7+
8+
public function setParameters(array $parameters) {
9+
$this->parameters = $parameters;
10+
return $this;
11+
}
12+
13+
public function getParameter($key, $default = null) {
14+
return idx($this->parameters, $key, $default);
15+
}
16+
17+
final public function getExtensionKey() {
18+
return $this->getPhobjectClassConstant('EXTENSIONKEY');
19+
}
20+
21+
final protected function getViewer() {
22+
return PhabricatorUser::getOmnipotentUser();
23+
}
24+
25+
abstract public function getExtensionName();
26+
27+
abstract public function shouldIndexObject($object);
28+
29+
abstract public function indexObject(
30+
PhabricatorIndexEngine $engine,
31+
$object);
32+
33+
public function getIndexVersion($object) {
34+
return null;
35+
}
36+
37+
final public static function getAllExtensions() {
38+
return id(new PhutilClassMapQuery())
39+
->setAncestorClass(__CLASS__)
40+
->setUniqueMethod('getExtensionKey')
41+
->execute();
42+
}
43+
44+
final public function shouldForceFullReindex() {
45+
return $this->getParameter('force');
46+
}
47+
48+
}

‎src/applications/search/index/PhabricatorIndexEngineExtensionModule.php

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<?php
2+
3+
final class PhabricatorIndexEngineExtensionModule
4+
extends PhabricatorConfigModule {
5+
6+
public function getModuleKey() {
7+
return 'indexengine';
8+
}
9+
10+
public function getModuleName() {
11+
return pht('Engine: Index');
12+
}
13+
14+
public function renderModuleStatus(AphrontRequest $request) {
15+
$viewer = $request->getViewer();
16+
17+
$extensions = PhabricatorIndexEngineExtension::getAllExtensions();
18+
19+
$rows = array();
20+
foreach ($extensions as $extension) {
21+
$rows[] = array(
22+
get_class($extension),
23+
$extension->getExtensionName(),
24+
);
25+
}
26+
27+
$table = id(new AphrontTableView($rows))
28+
->setHeaders(
29+
array(
30+
pht('Class'),
31+
pht('Name'),
32+
))
33+
->setColumnClasses(
34+
array(
35+
null,
36+
'wide pri',
37+
));
38+
39+
return id(new PHUIObjectBoxView())
40+
->setHeaderText(pht('IndexEngine Extensions'))
41+
->setTable($table);
42+
}
43+
44+
}

‎src/applications/search/index/PhabricatorSearchDocumentIndexer.php

+1-14
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,6 @@
22

33
abstract class PhabricatorSearchDocumentIndexer extends Phobject {
44

5-
private $context;
6-
7-
protected function setContext($context) {
8-
$this->context = $context;
9-
return $this;
10-
}
11-
12-
protected function getContext() {
13-
return $this->context;
14-
}
15-
165
abstract public function getIndexableObject();
176
abstract protected function buildAbstractDocumentByPHID($phid);
187

@@ -41,9 +30,7 @@ protected function loadDocumentByPHID($phid) {
4130
return $object;
4231
}
4332

44-
public function indexDocumentByPHID($phid, $context) {
45-
$this->setContext($context);
46-
33+
public function indexDocumentByPHID($phid) {
4734
$document = $this->buildAbstractDocumentByPHID($phid);
4835
if ($document === null) {
4936
// This indexer doesn't build a document index, so we're done.

0 commit comments

Comments
 (0)
Failed to load comments.