Skip to content

Commit e2296a0

Browse files
author
epriestley
committed
Modernize file storage engine selection
Summary: Fixes T5843. File storage engines use a very old "selector" mechanism which makes them difficult to extend. This mechanism predates widespread use of `PhutilSymbolLoader` to discover available implementations at runtime. Runtime discovery has generally proven more flexible and easier to use than explicit selection (although it sometimes needs more UI to support it in cases where order or enabled/disabled flags can not be directly determined). Use a modern runtime discovery mechanism instead of an explicit selector. This might break any installs which subclassed the `Selector`, but I believe almost no such installs exist, and they'll receive a meaningful exception upon upgrading (any custom engines will no longer implement all of the required methods). Looking forward, this modernizes infrastructure to prepare for new "virtual" chunked-storage engines, with the eventual goal of supporting very large file uploads and data import into the Phacility cluster. This uses D12051 to add UI to make it easier to understand the state of storage engines. Test Plan: Used new UI panel to assess storage engines: {F336270} - Uploaded a small file, saw it go to MySQL engine. - Uploaded a larger file, saw it go to S3 engine. Reviewers: btrahan Reviewed By: btrahan Subscribers: epriestley Maniphest Tasks: T5843 Differential Revision: https://secure.phabricator.com/D12053
1 parent 973079a commit e2296a0

15 files changed

+422
-247
lines changed

‎resources/celerity/map.php

Lines changed: 52 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
*/
88
return array(
99
'names' => array(
10-
'core.pkg.css' => '70320e8a',
10+
'core.pkg.css' => 'efdeeb14',
1111
'core.pkg.js' => '5f50c01b',
1212
'darkconsole.pkg.js' => '8ab24e01',
1313
'differential.pkg.css' => '1940be3f',
@@ -34,7 +34,7 @@
3434
'rsrc/css/aphront/typeahead.css' => '0e403212',
3535
'rsrc/css/application/almanac/almanac.css' => 'dbb9b3af',
3636
'rsrc/css/application/auth/auth.css' => '1e655982',
37-
'rsrc/css/application/base/main-menu-view.css' => 'f9f5cd1b',
37+
'rsrc/css/application/base/main-menu-view.css' => '58db7ad2',
3838
'rsrc/css/application/base/notification-menu.css' => '6aa0a74b',
3939
'rsrc/css/application/base/phabricator-application-launch-view.css' => '16ca323f',
4040
'rsrc/css/application/base/standard-page-view.css' => 'df338a4b',
@@ -44,7 +44,7 @@
4444
'rsrc/css/application/config/config-welcome.css' => '6abd79be',
4545
'rsrc/css/application/config/setup-issue.css' => '22270af2',
4646
'rsrc/css/application/config/unhandled-exception.css' => '37d4f9a2',
47-
'rsrc/css/application/conpherence/durable-column.css' => '7abcc3f2',
47+
'rsrc/css/application/conpherence/durable-column.css' => 'acefcb30',
4848
'rsrc/css/application/conpherence/menu.css' => 'c6ac5299',
4949
'rsrc/css/application/conpherence/message-pane.css' => '5930260a',
5050
'rsrc/css/application/conpherence/notification.css' => '04a6e10a',
@@ -105,12 +105,12 @@
105105
'rsrc/css/application/tokens/tokens.css' => '3d0f239e',
106106
'rsrc/css/application/uiexample/example.css' => '528b19de',
107107
'rsrc/css/core/core.css' => '86bfbe8c',
108-
'rsrc/css/core/remarkup.css' => '2dbff225',
108+
'rsrc/css/core/remarkup.css' => 'bc65f3cc',
109109
'rsrc/css/core/syntax.css' => '56c1ba38',
110110
'rsrc/css/core/z-index.css' => '2db67397',
111111
'rsrc/css/diviner/diviner-shared.css' => '38813222',
112112
'rsrc/css/font/font-awesome.css' => 'ae9a7b4d',
113-
'rsrc/css/font/font-source-sans-pro.css' => '0d859f60',
113+
'rsrc/css/font/font-source-sans-pro.css' => '4a2430d7',
114114
'rsrc/css/font/phui-font-icon-base.css' => '3dad2ae3',
115115
'rsrc/css/layout/phabricator-filetree-view.css' => 'fccf9f82',
116116
'rsrc/css/layout/phabricator-hovercard-view.css' => '893f4783',
@@ -128,7 +128,7 @@
128128
'rsrc/css/phui/phui-crumbs-view.css' => '594d719e',
129129
'rsrc/css/phui/phui-document.css' => '0f83a7df',
130130
'rsrc/css/phui/phui-feed-story.css' => 'c9f3a0b5',
131-
'rsrc/css/phui/phui-fontkit.css' => 'd30f4fa3',
131+
'rsrc/css/phui/phui-fontkit.css' => '1fa79503',
132132
'rsrc/css/phui/phui-form-view.css' => '78d729fe',
133133
'rsrc/css/phui/phui-form.css' => 'f535f938',
134134
'rsrc/css/phui/phui-header-view.css' => '083669db',
@@ -352,9 +352,9 @@
352352
'rsrc/js/application/aphlict/behavior-aphlict-status.js' => 'ea681761',
353353
'rsrc/js/application/auth/behavior-persona-login.js' => '9414ff18',
354354
'rsrc/js/application/config/behavior-reorder-fields.js' => '14a827de',
355-
'rsrc/js/application/conpherence/ConpherenceThreadManager.js' => 'efef202b',
356-
'rsrc/js/application/conpherence/behavior-durable-column.js' => 'aa3b6c22',
357-
'rsrc/js/application/conpherence/behavior-menu.js' => 'e476c952',
355+
'rsrc/js/application/conpherence/ConpherenceThreadManager.js' => '0324970d',
356+
'rsrc/js/application/conpherence/behavior-durable-column.js' => '9142e483',
357+
'rsrc/js/application/conpherence/behavior-menu.js' => 'c4151295',
358358
'rsrc/js/application/conpherence/behavior-pontificate.js' => '21ba5861',
359359
'rsrc/js/application/conpherence/behavior-quicksand-blacklist.js' => '7927a7d3',
360360
'rsrc/js/application/conpherence/behavior-widget-pane.js' => '2c1cd7f5',
@@ -514,11 +514,11 @@
514514
'changeset-view-manager' => '88be0133',
515515
'config-options-css' => '7fedf08b',
516516
'config-welcome-css' => '6abd79be',
517-
'conpherence-durable-column-view' => '7abcc3f2',
517+
'conpherence-durable-column-view' => 'acefcb30',
518518
'conpherence-menu-css' => 'c6ac5299',
519519
'conpherence-message-pane-css' => '5930260a',
520520
'conpherence-notification-css' => '04a6e10a',
521-
'conpherence-thread-manager' => 'efef202b',
521+
'conpherence-thread-manager' => '0324970d',
522522
'conpherence-update-css' => '1099a660',
523523
'conpherence-widget-pane-css' => '3d575438',
524524
'differential-changeset-view-css' => '6a8b172a',
@@ -534,7 +534,7 @@
534534
'diffusion-source-css' => '66fdf661',
535535
'diviner-shared-css' => '38813222',
536536
'font-fontawesome' => 'ae9a7b4d',
537-
'font-source-sans-pro' => '0d859f60',
537+
'font-source-sans-pro' => '4a2430d7',
538538
'global-drag-and-drop-css' => '697324ad',
539539
'harbormaster-css' => '49d64eb4',
540540
'herald-css' => '826075fa',
@@ -558,7 +558,7 @@
558558
'javelin-behavior-boards-dropdown' => '0ec56e1d',
559559
'javelin-behavior-choose-control' => '6153c708',
560560
'javelin-behavior-config-reorder-fields' => '14a827de',
561-
'javelin-behavior-conpherence-menu' => 'e476c952',
561+
'javelin-behavior-conpherence-menu' => 'c4151295',
562562
'javelin-behavior-conpherence-pontificate' => '21ba5861',
563563
'javelin-behavior-conpherence-widget-pane' => '2c1cd7f5',
564564
'javelin-behavior-countdown-timer' => 'e4cc26b3',
@@ -585,7 +585,7 @@
585585
'javelin-behavior-diffusion-locate-file' => '6d3e1947',
586586
'javelin-behavior-diffusion-pull-lastmodified' => '2b228192',
587587
'javelin-behavior-doorkeeper-tag' => 'e5822781',
588-
'javelin-behavior-durable-column' => 'aa3b6c22',
588+
'javelin-behavior-durable-column' => '9142e483',
589589
'javelin-behavior-error-log' => '6882e80a',
590590
'javelin-behavior-fancy-datepicker' => 'c51ae228',
591591
'javelin-behavior-global-drag-and-drop' => '07f199d8',
@@ -730,7 +730,7 @@
730730
'phabricator-hovercard-view-css' => '893f4783',
731731
'phabricator-keyboard-shortcut' => '1ae869f2',
732732
'phabricator-keyboard-shortcut-manager' => 'c1700f6f',
733-
'phabricator-main-menu-view' => 'f9f5cd1b',
733+
'phabricator-main-menu-view' => '58db7ad2',
734734
'phabricator-nav-view-css' => '7aeaf435',
735735
'phabricator-notification' => '0c6946e7',
736736
'phabricator-notification-css' => '9c279160',
@@ -739,7 +739,7 @@
739739
'phabricator-phtize' => 'd254d646',
740740
'phabricator-prefab' => '72da38cc',
741741
'phabricator-profile-css' => '1a20dcbf',
742-
'phabricator-remarkup-css' => '2dbff225',
742+
'phabricator-remarkup-css' => 'bc65f3cc',
743743
'phabricator-search-results-css' => '559cc554',
744744
'phabricator-shaped-request' => '7cbe244b',
745745
'phabricator-side-menu-view-css' => '7e8c6341',
@@ -783,7 +783,7 @@
783783
'phui-document-view-css' => '0f83a7df',
784784
'phui-feed-story-css' => 'c9f3a0b5',
785785
'phui-font-icon-base-css' => '3dad2ae3',
786-
'phui-fontkit-css' => 'd30f4fa3',
786+
'phui-fontkit-css' => '1fa79503',
787787
'phui-form-css' => 'f535f938',
788788
'phui-form-view-css' => '78d729fe',
789789
'phui-header-view-css' => '083669db',
@@ -845,6 +845,16 @@
845845
'029a133d' => array(
846846
'aphront-dialog-view-css',
847847
),
848+
'0324970d' => array(
849+
'javelin-dom',
850+
'javelin-util',
851+
'javelin-stratcom',
852+
'javelin-install',
853+
'javelin-workflow',
854+
'javelin-router',
855+
'javelin-behavior-device',
856+
'javelin-vector',
857+
),
848858
'03d6ed07' => array(
849859
'javelin-behavior',
850860
'javelin-stratcom',
@@ -1141,6 +1151,9 @@
11411151
'javelin-request',
11421152
'javelin-util',
11431153
),
1154+
'4a2430d7' => array(
1155+
'phui-fontkit-css',
1156+
),
11441157
'4d94d9c3' => array(
11451158
'javelin-behavior',
11461159
'javelin-stratcom',
@@ -1533,6 +1546,16 @@
15331546
'javelin-uri',
15341547
'phabricator-notification',
15351548
),
1549+
'9142e483' => array(
1550+
'javelin-behavior',
1551+
'javelin-dom',
1552+
'javelin-stratcom',
1553+
'javelin-behavior-device',
1554+
'javelin-scrollbar',
1555+
'javelin-quicksand',
1556+
'phabricator-keyboard-shortcut',
1557+
'conpherence-thread-manager',
1558+
),
15361559
'92eb531d' => array(
15371560
'javelin-behavior',
15381561
'javelin-dom',
@@ -1651,15 +1674,6 @@
16511674
'javelin-util',
16521675
'phabricator-prefab',
16531676
),
1654-
'aa3b6c22' => array(
1655-
'javelin-behavior',
1656-
'javelin-dom',
1657-
'javelin-stratcom',
1658-
'javelin-scrollbar',
1659-
'javelin-quicksand',
1660-
'phabricator-keyboard-shortcut',
1661-
'conpherence-thread-manager',
1662-
),
16631677
'b1f0ccee' => array(
16641678
'javelin-install',
16651679
'javelin-dom',
@@ -1744,6 +1758,18 @@
17441758
'javelin-dom',
17451759
'javelin-vector',
17461760
),
1761+
'c4151295' => array(
1762+
'javelin-behavior',
1763+
'javelin-dom',
1764+
'javelin-util',
1765+
'javelin-stratcom',
1766+
'javelin-workflow',
1767+
'javelin-behavior-device',
1768+
'javelin-history',
1769+
'javelin-vector',
1770+
'phabricator-shaped-request',
1771+
'conpherence-thread-manager',
1772+
),
17471773
'c51ae228' => array(
17481774
'javelin-behavior',
17491775
'javelin-util',
@@ -1847,18 +1873,6 @@
18471873
'javelin-dom',
18481874
'javelin-uri',
18491875
),
1850-
'e476c952' => array(
1851-
'javelin-behavior',
1852-
'javelin-dom',
1853-
'javelin-util',
1854-
'javelin-stratcom',
1855-
'javelin-workflow',
1856-
'javelin-behavior-device',
1857-
'javelin-history',
1858-
'javelin-vector',
1859-
'phabricator-shaped-request',
1860-
'conpherence-thread-manager',
1861-
),
18621876
'e4cc26b3' => array(
18631877
'javelin-behavior',
18641878
'javelin-dom',
@@ -1906,16 +1920,6 @@
19061920
'javelin-install',
19071921
'javelin-util',
19081922
),
1909-
'efef202b' => array(
1910-
'javelin-dom',
1911-
'javelin-util',
1912-
'javelin-stratcom',
1913-
'javelin-install',
1914-
'javelin-workflow',
1915-
'javelin-router',
1916-
'javelin-behavior-device',
1917-
'javelin-vector',
1918-
),
19191923
'f24f3253' => array(
19201924
'javelin-behavior',
19211925
'javelin-dom',

‎src/__phutil_library_map__.php

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1689,7 +1689,6 @@
16891689
'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
16901690
'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php',
16911691
'PhabricatorDebugController' => 'applications/system/controller/PhabricatorDebugController.php',
1692-
'PhabricatorDefaultFileStorageEngineSelector' => 'applications/files/engineselector/PhabricatorDefaultFileStorageEngineSelector.php',
16931692
'PhabricatorDefaultSearchEngineSelector' => 'applications/search/selector/PhabricatorDefaultSearchEngineSelector.php',
16941693
'PhabricatorDestructibleInterface' => 'applications/system/interface/PhabricatorDestructibleInterface.php',
16951694
'PhabricatorDestructionEngine' => 'applications/system/engine/PhabricatorDestructionEngine.php',
@@ -1813,7 +1812,6 @@
18131812
'PhabricatorFileStorageBlob' => 'applications/files/storage/PhabricatorFileStorageBlob.php',
18141813
'PhabricatorFileStorageConfigurationException' => 'applications/files/exception/PhabricatorFileStorageConfigurationException.php',
18151814
'PhabricatorFileStorageEngine' => 'applications/files/engine/PhabricatorFileStorageEngine.php',
1816-
'PhabricatorFileStorageEngineSelector' => 'applications/files/engineselector/PhabricatorFileStorageEngineSelector.php',
18171815
'PhabricatorFileTemporaryGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php',
18181816
'PhabricatorFileTestCase' => 'applications/files/storage/__tests__/PhabricatorFileTestCase.php',
18191817
'PhabricatorFileTestDataGenerator' => 'applications/files/lipsum/PhabricatorFileTestDataGenerator.php',
@@ -1826,6 +1824,7 @@
18261824
'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php',
18271825
'PhabricatorFileinfoSetupCheck' => 'applications/config/check/PhabricatorFileinfoSetupCheck.php',
18281826
'PhabricatorFilesApplication' => 'applications/files/application/PhabricatorFilesApplication.php',
1827+
'PhabricatorFilesApplicationStorageEnginePanel' => 'applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php',
18291828
'PhabricatorFilesConfigOptions' => 'applications/files/config/PhabricatorFilesConfigOptions.php',
18301829
'PhabricatorFilesManagementCompactWorkflow' => 'applications/files/management/PhabricatorFilesManagementCompactWorkflow.php',
18311830
'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
@@ -4977,7 +4976,6 @@
49774976
'PhabricatorDataNotAttachedException' => 'Exception',
49784977
'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck',
49794978
'PhabricatorDebugController' => 'PhabricatorController',
4980-
'PhabricatorDefaultFileStorageEngineSelector' => 'PhabricatorFileStorageEngineSelector',
49814979
'PhabricatorDefaultSearchEngineSelector' => 'PhabricatorSearchEngineSelector',
49824980
'PhabricatorDestructionEngine' => 'Phobject',
49834981
'PhabricatorDeveloperConfigOptions' => 'PhabricatorApplicationConfigOptions',
@@ -5122,6 +5120,7 @@
51225120
'PhabricatorFileUploadException' => 'Exception',
51235121
'PhabricatorFileinfoSetupCheck' => 'PhabricatorSetupCheck',
51245122
'PhabricatorFilesApplication' => 'PhabricatorApplication',
5123+
'PhabricatorFilesApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel',
51255124
'PhabricatorFilesConfigOptions' => 'PhabricatorApplicationConfigOptions',
51265125
'PhabricatorFilesManagementCompactWorkflow' => 'PhabricatorFilesManagementWorkflow',
51275126
'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',

‎src/applications/config/check/PhabricatorExtraConfigSetupCheck.php

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,9 @@ public static function getAncientConfig() {
211211
'phd.start-taskmasters' => pht(
212212
'Taskmasters now use an autoscaling pool. You can configure the '.
213213
'pool size with `phd.taskmasters`.'),
214+
'storage.engine-selector' => pht(
215+
'Phabricator now automatically discovers available storage engines '.
216+
'at runtime.'),
214217
);
215218

216219
return $ancient_config;

‎src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
<?php
2+
3+
final class PhabricatorFilesApplicationStorageEnginePanel
4+
extends PhabricatorApplicationConfigurationPanel {
5+
6+
public function getPanelKey() {
7+
return 'storage';
8+
}
9+
10+
public function shouldShowForApplication(
11+
PhabricatorApplication $application) {
12+
return ($application instanceof PhabricatorFilesApplication);
13+
}
14+
15+
public function buildConfigurationPagePanel() {
16+
$viewer = $this->getViewer();
17+
$application = $this->getApplication();
18+
19+
$engines = PhabricatorFileStorageEngine::loadAllEngines();
20+
$writable_engines = PhabricatorFileStorageEngine::loadWritableEngines();
21+
22+
$yes = pht('Yes');
23+
$no = pht('No');
24+
25+
$rows = array();
26+
$rowc = array();
27+
foreach ($engines as $key => $engine) {
28+
$limited = $no;
29+
$limit = null;
30+
if ($engine->hasFilesizeLimit()) {
31+
$limited = $yes;
32+
$limit = phutil_format_bytes($engine->getFilesizeLimit());
33+
}
34+
35+
if ($engine->canWriteFiles()) {
36+
$writable = $yes;
37+
} else {
38+
$writable = $no;
39+
}
40+
41+
if ($engine->isTestEngine()) {
42+
$test = $yes;
43+
} else {
44+
$test = $no;
45+
}
46+
47+
if (isset($writable_engines[$key])) {
48+
$rowc[] = 'highlighted';
49+
} else {
50+
$rowc[] = null;
51+
}
52+
53+
$rows[] = array(
54+
$key,
55+
get_class($engine),
56+
$test,
57+
$writable,
58+
$limited,
59+
$limit,
60+
);
61+
}
62+
63+
$table =
64+
65+
$table = id(new AphrontTableView($rows))
66+
->setNoDataString(pht('No storage engines available.'))
67+
->setHeaders(
68+
array(
69+
pht('Key'),
70+
pht('Class'),
71+
pht('Unit Test'),
72+
pht('Writable'),
73+
pht('Has Limit'),
74+
pht('Limit'),
75+
))
76+
->setRowClasses($rowc)
77+
->setColumnClasses(
78+
array(
79+
'',
80+
'wide',
81+
'',
82+
'',
83+
'',
84+
'n',
85+
));
86+
87+
$box = id(new PHUIObjectBoxView())
88+
->setHeaderText(pht('Storage Engines'))
89+
->appendChild($table);
90+
91+
return $box;
92+
}
93+
94+
public function handlePanelRequest(
95+
AphrontRequest $request,
96+
PhabricatorController $controller) {
97+
return new Aphront404Response();
98+
}
99+
100+
}

0 commit comments

Comments
 (0)