Skip to content

Commit 5d1bd51

Browse files
author
epriestley
committed
Add a script to migrate files between storage engines
Summary: Quora requested this (moving to S3) but it's also clearly a good idea. Test Plan: Ran with various valid/invalid options to test options. Error/sanity checking seemed OK. Migrated individual local files. Migrated all my local files back and forth between engines several times. Uploaded some new files. Reviewers: btrahan, vrana Reviewed By: vrana CC: aran Maniphest Tasks: T1950 Differential Revision: https://secure.phabricator.com/D3808
1 parent 92a2866 commit 5d1bd51

File tree

8 files changed

+372
-27
lines changed

8 files changed

+372
-27
lines changed

‎bin/files

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../scripts/files/manage_files.php

‎scripts/files/manage_files.php

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env php
2+
<?php
3+
4+
/*
5+
* Copyright 2012 Facebook, Inc.
6+
*
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
20+
$root = dirname(dirname(dirname(__FILE__)));
21+
require_once $root.'/scripts/__init_script__.php';
22+
23+
$args = new PhutilArgumentParser($argv);
24+
$args->setTagline('manage files');
25+
$args->setSynopsis(<<<EOSYNOPSIS
26+
**files** __command__ [__options__]
27+
Manage Phabricator file storage.
28+
29+
EOSYNOPSIS
30+
);
31+
$args->parseStandardArguments();
32+
33+
$workflows = array(
34+
new PhabricatorFilesManagementEnginesWorkflow(),
35+
new PhabricatorFilesManagementMigrateWorkflow(),
36+
new PhutilHelpArgumentWorkflow(),
37+
);
38+
39+
$args->parseWorkflows($workflows);

‎src/__phutil_library_map__.php

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,6 +755,9 @@
755755
'PhabricatorFileUploadController' => 'applications/files/controller/PhabricatorFileUploadController.php',
756756
'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php',
757757
'PhabricatorFileUploadView' => 'applications/files/view/PhabricatorFileUploadView.php',
758+
'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
759+
'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php',
760+
'PhabricatorFilesManagementWorkflow' => 'applications/files/management/PhabricatorFilesManagementWorkflow.php',
758761
'PhabricatorFlag' => 'applications/flag/storage/PhabricatorFlag.php',
759762
'PhabricatorFlagColor' => 'applications/flag/constants/PhabricatorFlagColor.php',
760763
'PhabricatorFlagConstants' => 'applications/flag/constants/PhabricatorFlagConstants.php',
@@ -1949,6 +1952,9 @@
19491952
'PhabricatorFileUploadController' => 'PhabricatorFileController',
19501953
'PhabricatorFileUploadException' => 'Exception',
19511954
'PhabricatorFileUploadView' => 'AphrontView',
1955+
'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',
1956+
'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow',
1957+
'PhabricatorFilesManagementWorkflow' => 'PhutilArgumentWorkflow',
19521958
'PhabricatorFlag' => 'PhabricatorFlagDAO',
19531959
'PhabricatorFlagColor' => 'PhabricatorFlagConstants',
19541960
'PhabricatorFlagController' => 'PhabricatorController',

‎src/applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
<?php
2+
3+
/*
4+
* Copyright 2012 Facebook, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License");
7+
* you may not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
final class PhabricatorFilesManagementEnginesWorkflow
20+
extends PhabricatorFilesManagementWorkflow {
21+
22+
public function didConstruct() {
23+
$this
24+
->setName('engines')
25+
->setSynopsis('List available storage engines.')
26+
->setArguments(array());
27+
}
28+
29+
public function execute(PhutilArgumentParser $args) {
30+
$console = PhutilConsole::getConsole();
31+
32+
$engines = PhabricatorFile::buildAllEngines();
33+
if (!$engines) {
34+
throw new Exception("No storage engines are available.");
35+
}
36+
37+
foreach ($engines as $engine) {
38+
$console->writeOut(
39+
"%s\n",
40+
$engine->getEngineIdentifier());
41+
}
42+
43+
return 0;
44+
}
45+
46+
}

‎src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
<?php
2+
3+
/*
4+
* Copyright 2012 Facebook, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License");
7+
* you may not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
final class PhabricatorFilesManagementMigrateWorkflow
20+
extends PhabricatorFilesManagementWorkflow {
21+
22+
public function didConstruct() {
23+
$this
24+
->setName('migrate')
25+
->setSynopsis('Migrate files between storage engines.')
26+
->setArguments(
27+
array(
28+
array(
29+
'name' => 'engine',
30+
'param' => 'storage_engine',
31+
'help' => 'Migrate to the named storage engine.',
32+
),
33+
array(
34+
'name' => 'dry-run',
35+
'help' => 'Show what would be migrated.',
36+
),
37+
array(
38+
'name' => 'all',
39+
'help' => 'Migrate all files.',
40+
),
41+
array(
42+
'name' => 'names',
43+
'wildcard' => true,
44+
),
45+
));
46+
}
47+
48+
public function execute(PhutilArgumentParser $args) {
49+
$console = PhutilConsole::getConsole();
50+
51+
$engine_id = $args->getArg('engine');
52+
if (!$engine_id) {
53+
throw new PhutilArgumentUsageException(
54+
"Specify an engine to migrate to with `--engine`. ".
55+
"Use `files engines` to get a list of engines.");
56+
}
57+
58+
$engine = PhabricatorFile::buildEngine($engine_id);
59+
60+
if ($args->getArg('all')) {
61+
if ($args->getArg('names')) {
62+
throw new PhutilArgumentUsageException(
63+
"Specify either a list of files or `--all`, but not both.");
64+
}
65+
$iterator = new LiskMigrationIterator(new PhabricatorFile());
66+
} else if ($args->getArg('names')) {
67+
$iterator = array();
68+
69+
foreach ($args->getArg('names') as $name) {
70+
$name = trim($name);
71+
72+
$id = preg_replace('/^F/i', '', $name);
73+
if (ctype_digit($id)) {
74+
$file = id(new PhabricatorFile())->loadOneWhere(
75+
'id = %d',
76+
$id);
77+
if (!$file) {
78+
throw new PhutilArgumentUsageException(
79+
"No file exists with id '{$name}'.");
80+
}
81+
} else {
82+
$file = id(new PhabricatorFile())->loadOneWhere(
83+
'phid = %d',
84+
$name);
85+
if (!$file) {
86+
throw new PhutilArgumentUsageException(
87+
"No file exists with PHID '{$name}'.");
88+
}
89+
}
90+
$iterator[] = $file;
91+
}
92+
} else {
93+
throw new PhutilArgumentUsageException(
94+
"Either specify a list of files to migrate, or use `--all` ".
95+
"to migrate all files.");
96+
}
97+
98+
$is_dry_run = $args->getArg('dry-run');
99+
100+
$failed = array();
101+
102+
foreach ($iterator as $file) {
103+
$fid = 'F'.$file->getID();
104+
105+
if ($file->getStorageEngine() == $engine_id) {
106+
$console->writeOut(
107+
"%s: Already stored on '%s'\n",
108+
$fid,
109+
$engine_id);
110+
continue;
111+
}
112+
113+
if ($is_dry_run) {
114+
$console->writeOut(
115+
"%s: Would migrate from '%s' to '%s' (dry run)\n",
116+
$fid,
117+
$file->getStorageEngine(),
118+
$engine_id);
119+
continue;
120+
}
121+
122+
$console->writeOut(
123+
"%s: Migrating from '%s' to '%s'...",
124+
$fid,
125+
$file->getStorageEngine(),
126+
$engine_id);
127+
128+
try {
129+
$file->migrateToEngine($engine);
130+
$console->writeOut("done.\n");
131+
} catch (Exception $ex) {
132+
$console->writeOut("failed!\n");
133+
$console->writeErr("%s\n", (string)$ex);
134+
$failed[] = $file;
135+
}
136+
}
137+
138+
if ($failed) {
139+
$console->writeOut("**Failures!**\n");
140+
$ids = array();
141+
foreach ($failed as $file) {
142+
$ids[] = 'F'.$file->getID();
143+
}
144+
$console->writeOut("%s\n", implode(', ', $ids));
145+
146+
return 1;
147+
} else {
148+
$console->writeOut("**Success!**\n");
149+
return 0;
150+
}
151+
}
152+
153+
}

‎src/applications/files/management/PhabricatorFilesManagementWorkflow.php

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<?php
2+
3+
/*
4+
* Copyright 2012 Facebook, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License");
7+
* you may not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
abstract class PhabricatorFilesManagementWorkflow
20+
extends PhutilArgumentWorkflow {
21+
22+
public function isExecutable() {
23+
return true;
24+
}
25+
26+
}

0 commit comments

Comments
 (0)