Skip to content

Commit a15130b

Browse files
author
epriestley
committed
Add a maintenance script for reconciling repositories to disk state
Summary: @rguerin ran into an issue in his install where Phabricator appears to have discovered commits which no longer exist, and thus is failing to proceed with its repository import. It's not clear how we got into this state. Previously, it was possible by, e.g., parsing a different repository's working copy and then switching them back, but there are now safeguards against that. I'm taking a three-pronged approach to try to sort this out: - Provide a script to get out of this state (this script) and reconcile Phabricator's view of a repository with an authoritative copy of it. This basically "un-discovers" any discovered commits which don't actually exist (any queued tasks to parse them will fail permanently when they fail to load the commit object). - Add more logging to the discovery daemon so we can figure out where commits came from. - Improve Diffusion's UI when stuff is partially discovered (T776). (This script should also clean up some nonsense on secure.phabricator.com from a botched Diviner import.) Test Plan: Ran "reconcile.php" with bogus commits and bogus differential/commit links, had them expunged. Will work with @rguerin to see if this resolves things. Reviewers: btrahan, rguerin Reviewed By: btrahan CC: aran, epriestley Differential Revision: https://secure.phabricator.com/D1552
1 parent c86dfd8 commit a15130b

File tree

3 files changed

+191
-1
lines changed

3 files changed

+191
-1
lines changed

Diff for: scripts/repository/reconcile.php

+165
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
#!/usr/bin/env php
2+
<?php
3+
4+
/*
5+
* Copyright 2012 Facebook, Inc.
6+
*
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
20+
$root = dirname(dirname(dirname(__FILE__)));
21+
require_once $root.'/scripts/__init_script__.php';
22+
23+
$args = new PhutilArgumentParser($argv);
24+
$args->setTagline('reconcile Phabricator state after repository changes');
25+
$args->setSynopsis(<<<EOSYNOPSIS
26+
**reconcile.php** __repository_callsign__
27+
Reconcile the state of Phabricator's caches with the actual state
28+
of the repository.
29+
30+
This is an administrative/maintenace operation and not generally
31+
necessary, but if repository history has changed or been rewritten
32+
(for example, if the repository was stored from a backup)
33+
Phabricator may think commits which are no longer present in the
34+
repository still exist.
35+
36+
This will delete all evidence of commits which Phabricator can't
37+
find in the actual repository.
38+
39+
EOSYNOPSIS
40+
);
41+
$args->parseStandardArguments();
42+
$args->parse(
43+
array(
44+
array(
45+
'name' => 'more',
46+
'wildcard' => true,
47+
),
48+
));
49+
50+
$more = $args->getArg('more');
51+
if (count($more) !== 1) {
52+
$args->printHelpAndExit();
53+
}
54+
$callsign = reset($more);
55+
56+
57+
$repository = id(new PhabricatorRepository())->loadOneWhere(
58+
'callsign = %s',
59+
$callsign);
60+
if (!$repository) {
61+
throw new Exception("No repository exists with callsign '{$callsign}'!");
62+
}
63+
64+
switch ($repository->getVersionControlSystem()) {
65+
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
66+
break;
67+
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
68+
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
69+
default:
70+
throw new Exception("For now, you can only reconcile git repositories.");
71+
}
72+
73+
echo "Loading commits...\n";
74+
$all_commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
75+
'repositoryID = %d',
76+
$repository->getID());
77+
78+
echo "Updating repository..\n";
79+
try {
80+
// Sanity-check the repository working copy and make sure we're up to date.
81+
$repository->execxLocalCommand('fetch --all');
82+
} catch (Exception $ex) {
83+
echo "Unable to `git fetch` the working copy to update it. Reconciliation ".
84+
"requires an up-to-date working copy.\n";
85+
throw $ex;
86+
}
87+
88+
echo "Verifying commits (this may take some time if the repository is large)";
89+
$futures = array();
90+
foreach ($all_commits as $id => $commit) {
91+
$futures[$id] = $repository->getLocalCommandFuture(
92+
'rev-parse --verify %s',
93+
$commit->getCommitIdentifier());
94+
}
95+
96+
$bad = array();
97+
foreach (Futures($futures)->limit(8) as $id => $future) {
98+
list($err) = $future->resolve();
99+
if ($err) {
100+
$bad[$id] = $all_commits[$id];
101+
echo "#";
102+
} else {
103+
echo ".";
104+
}
105+
}
106+
echo "\nDone.\n";
107+
108+
if (!count($bad)) {
109+
echo "No bad commits found!\n";
110+
} else {
111+
echo "Found ".count($bad)." bad commits:\n\n";
112+
echo ' '.implode("\n ", mpull($bad, 'getCommitIdentifier'));
113+
$ok = phutil_console_confirm("Do you want to delete these commits?");
114+
if (!$ok) {
115+
echo "OK, aborting.\n";
116+
exit(1);
117+
}
118+
119+
echo "Deleting commits";
120+
foreach ($bad as $commit) {
121+
echo ".";
122+
$commit->delete();
123+
}
124+
echo "\nDone.\n";
125+
}
126+
127+
//// Clean Up Links ////////////////////////////////////////////////////////
128+
129+
$table = new PhabricatorRepositoryCommit();
130+
131+
$valid_phids = queryfx_all(
132+
$table->establishConnection('r'),
133+
'SELECT phid FROM %T',
134+
$table->getTableName());
135+
$valid_phids = ipull($valid_phids, null, 'phid');
136+
137+
//////// Differential <-> Diffusion Links //////////////////////////////////
138+
139+
$dx_conn = id(new DifferentialRevision())->establishConnection('w');
140+
$dx_table = DifferentialRevision::TABLE_COMMIT;
141+
$dx_phids = queryfx_all(
142+
$dx_conn,
143+
'SELECT commitPHID FROM %T',
144+
$dx_table);
145+
146+
$bad_phids = array();
147+
foreach ($dx_phids as $dx_phid) {
148+
if (empty($valid_phids[$dx_phid['commitPHID']])) {
149+
$bad_phids[] = $dx_phid['commitPHID'];
150+
}
151+
}
152+
153+
if ($bad_phids) {
154+
echo "Deleting ".count($bad_phids)." bad Diffusion links...\n";
155+
queryfx(
156+
$dx_conn,
157+
'DELETE FROM %T WHERE commitPHID IN (%Ls)',
158+
$dx_table,
159+
$bad_phids);
160+
echo "Done.\n";
161+
} else {
162+
echo "Diffusion links are clean.\n";
163+
}
164+
165+
// TODO: There are some links in owners that we should probably clean up too.

Diff for: src/applications/repository/storage/commit/PhabricatorRepositoryCommit.php

+23-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<?php
22

33
/*
4-
* Copyright 2011 Facebook, Inc.
4+
* Copyright 2012 Facebook, Inc.
55
*
66
* Licensed under the Apache License, Version 2.0 (the "License");
77
* you may not use this file except in compliance with the License.
@@ -37,6 +37,15 @@ public function generatePHID() {
3737
PhabricatorPHIDConstants::PHID_TYPE_CMIT);
3838
}
3939

40+
public function loadCommitData() {
41+
if (!$this->getID()) {
42+
return null;
43+
}
44+
return id(new PhabricatorRepositoryCommitData())->loadOneWhere(
45+
'commitID = %d',
46+
$this->getID());
47+
}
48+
4049
public function attachCommitData(PhabricatorRepositoryCommitData $data) {
4150
$this->commitData = $data;
4251
return $this;
@@ -49,4 +58,17 @@ public function getCommitData() {
4958
return $this->commitData;
5059
}
5160

61+
public function delete() {
62+
$data = $this->loadCommitData();
63+
$this->openTransaction();
64+
65+
if ($data) {
66+
$data->delete();
67+
}
68+
$result = parent::delete();
69+
70+
$this->saveTransaction();
71+
return $result;
72+
}
73+
5274
}

Diff for: src/applications/repository/storage/commit/__init__.php

+3
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
phutil_require_module('phabricator', 'applications/phid/constants');
1010
phutil_require_module('phabricator', 'applications/phid/storage/phid');
1111
phutil_require_module('phabricator', 'applications/repository/storage/base');
12+
phutil_require_module('phabricator', 'applications/repository/storage/commitdata');
13+
14+
phutil_require_module('phutil', 'utils');
1215

1316

1417
phutil_require_source('PhabricatorRepositoryCommit.php');

0 commit comments

Comments
 (0)