Skip to content

Commit 4893146

Browse files
author
epriestley
committed
Improve parser scalability, fix a bug or two, provide 'phd', the Phabricator
Daemon interface.
1 parent 383b3d7 commit 4893146

File tree

9 files changed

+191
-25
lines changed

9 files changed

+191
-25
lines changed

‎bin/phd‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../scripts/daemon/phabricator_daemon_launcher.php

‎scripts/__init_script__.php‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@
2525
exit(1);
2626
}
2727

28-
if (!ini_get('date.timezone')) {
29-
date_default_timezone_set('America/Los_Angeles');
30-
}
31-
3228
phutil_load_library(dirname(__FILE__).'/../src/');
29+
30+
phutil_require_module('phutil', 'symbols');
31+
32+
function __autoload($class) {
33+
PhutilSymbolLoader::loadClass($class);
34+
}

‎scripts/daemon/phabricator_daemon_launcher.php‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env php
2+
<?php
3+
4+
/*
5+
* Copyright 2011 Facebook, Inc.
6+
*
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
20+
$root = dirname(dirname(dirname(__FILE__)));
21+
require_once $root.'/scripts/__init_script__.php';
22+
require_once $root.'/scripts/__init_env__.php';
23+
24+
25+
26+
switch (isset($argv[1]) ? $argv[1] : 'help') {
27+
case 'parse-commit':
28+
$commit = isset($argv[2]) ? $argv[2] : null;
29+
if (!$commit) {
30+
throw new Exception("Provide a commit to parse!");
31+
}
32+
$matches = null;
33+
if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
34+
throw new Exception("Can't parse commit identifier!");
35+
}
36+
$repo = id(new PhabricatorRepository())->loadOneWhere(
37+
'callsign = %s',
38+
$matches[1]);
39+
if (!$repo) {
40+
throw new Exception("Unknown repository!");
41+
}
42+
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
43+
'repositoryID = %d AND commitIdentifier = %s',
44+
$repo->getID(),
45+
$matches[2]);
46+
if (!$commit) {
47+
throw new Exception('Unknown commit.');
48+
}
49+
50+
switch ($repo->getVersionControlSystem()) {
51+
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
52+
$worker = new PhabricatorRepositoryGitCommitChangeParserWorker(
53+
$commit->getID());
54+
break;
55+
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
56+
$worker = new PhabricatorRepositorySvnCommitChangeParserWorker(
57+
$commit->getID());
58+
break;
59+
default:
60+
throw new Exception("Unknown repository type!");
61+
}
62+
63+
ExecFuture::pushEchoMode(true);
64+
65+
$worker->doWork();
66+
67+
echo "Done.\n";
68+
69+
break;
70+
case '--help':
71+
case 'help':
72+
default:
73+
echo <<<EOHELP
74+
phd - phabricator daemon launcher
75+
76+
parse-commit <rXnnnn>
77+
Parse a single commit.
78+
79+
EOHELP;
80+
exit(1);
81+
}

‎src/applications/diffusion/controller/home/__init__.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@
77

88

99
phutil_require_module('phabricator', 'applications/diffusion/controller/base');
10+
phutil_require_module('phabricator', 'applications/diffusion/view/base');
1011
phutil_require_module('phabricator', 'applications/repository/storage/commit');
1112
phutil_require_module('phabricator', 'applications/repository/storage/repository');
1213
phutil_require_module('phabricator', 'storage/queryfx');
1314
phutil_require_module('phabricator', 'view/control/table');
1415
phutil_require_module('phabricator', 'view/layout/panel');
15-
phutil_require_module('phabricator', 'view/utils');
1616

1717
phutil_require_module('phutil', 'markup');
1818
phutil_require_module('phutil', 'utils');

‎src/applications/diffusion/controller/repository/__init__.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,7 @@
1515
phutil_require_module('phabricator', 'applications/diffusion/view/historytable');
1616
phutil_require_module('phabricator', 'view/layout/panel');
1717

18+
phutil_require_module('phutil', 'markup');
19+
1820

1921
phutil_require_source('DiffusionRepositoryController.php');

‎src/applications/diffusion/query/history/svn/__init__.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88

99
phutil_require_module('phabricator', 'applications/diffusion/data/pathchange');
1010
phutil_require_module('phabricator', 'applications/diffusion/query/history/base');
11+
phutil_require_module('phabricator', 'applications/repository/storage/commit');
12+
phutil_require_module('phabricator', 'applications/repository/storage/commitdata');
1113
phutil_require_module('phabricator', 'applications/repository/storage/repository');
1214
phutil_require_module('phabricator', 'storage/queryfx');
1315

‎src/applications/diffusion/view/historytable/__init__.php‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
phutil_require_module('phabricator', 'applications/diffusion/view/base');
1010
phutil_require_module('phabricator', 'view/control/table');
11-
phutil_require_module('phabricator', 'view/utils');
1211

1312
phutil_require_module('phutil', 'markup');
1413

‎src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php‎

Lines changed: 95 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,7 @@ private function lookupPathFileTypes(
516516
$parent = $repository_uri.$parent.'@'.$lookup['rawCommit'];
517517
$parent = escapeshellarg($parent);
518518
$parents[$parent] = true;
519-
$path_mapping[$parent][] = $path;
519+
$path_mapping[$parent][] = dirname($path);
520520
}
521521

522522
$result_map = array();
@@ -592,21 +592,104 @@ private function lookupRecursiveFileList(
592592
$rev = $info['rawCommit'];
593593
$path = $this->encodeSVNPath($path);
594594

595-
// TODO: This is a scalability nightmare.
595+
$hashkey = md5($repository->getDetail('remote-uri').$path.'@'.$rev);
596+
597+
// This method is quite horrible. The underlying challenge is that some
598+
// commits in the Facebook repository are enormous, taking multiple hours
599+
// to 'ls -R' out of the repository and producing XML files >1GB in size.
600+
601+
// If we try to SimpleXML them, the object exhausts available memory on a
602+
// 64G machine. Instead, cache the XML output and then parse it line by line
603+
// to limit space requirements.
604+
605+
$cache_loc = sys_get_temp_dir().'/diffusion.'.$hashkey.'.svnls';
606+
if (!Filesystem::pathExists($cache_loc)) {
607+
$tmp = new TempFile();
608+
execx(
609+
'svn --non-interactive --xml ls -R %s%s@%d > %s',
610+
$repository->getDetail('remote-uri'),
611+
$path,
612+
$rev,
613+
$tmp);
614+
execx(
615+
'mv %s %s',
616+
$tmp,
617+
$cache_loc);
618+
}
619+
620+
$map = $this->parseRecursiveListFileData($cache_loc);
621+
Filesystem::remove($cache_loc);
596622

597-
list($raw_xml) = execx(
598-
'svn --non-interactive --xml ls -R %s%s@%d',
599-
$repository->getDetail('remote-uri'),
600-
$path,
601-
$rev);
623+
return $map;
624+
}
602625

626+
private function parseRecursiveListFileData($file_path) {
603627
$map = array();
604628

605-
$xml = new SimpleXMLElement($raw_xml);
606-
foreach ($xml->list[0] as $entry) {
607-
$key = (string)$entry->name;
608-
$file_type = $this->getFileTypeFromSVNKind($entry['kind']);
609-
$map[$key] = $file_type;
629+
$mode = 'xml';
630+
$done = false;
631+
$entry = null;
632+
foreach (new LinesOfALargeFile($file_path) as $lno => $line) {
633+
switch ($mode) {
634+
case 'entry':
635+
if ($line == '</entry>') {
636+
$entry = implode('', $entry);
637+
$pattern = '@^\s+kind="(file|dir)">'.
638+
'<name>(.*?)</name>'.
639+
'(<size>(.*?)</size>)?@';
640+
$matches = null;
641+
if (!preg_match($pattern, $entry, $matches)) {
642+
throw new Exception("Unable to parse entry!");
643+
}
644+
$map[html_entity_decode($matches[2])] =
645+
$this->getFileTypeFromSVNKind($matches[1]);
646+
$mode = 'entry-or-end';
647+
} else {
648+
$entry[] = $line;
649+
}
650+
break;
651+
case 'entry-or-end':
652+
if ($line == '</list>') {
653+
$done = true;
654+
break 2;
655+
} else if ($line == '<entry') {
656+
$mode = 'entry';
657+
$entry = array();
658+
} else {
659+
throw new Exception("Expected </list> or <entry, got {$line}.");
660+
}
661+
break;
662+
case 'xml':
663+
$expect = '<?xml version="1.0"?>';
664+
if ($line !== $expect) {
665+
throw new Exception("Expected '{$expect}', got {$line}.");
666+
}
667+
$mode = 'list';
668+
break;
669+
case 'list':
670+
$expect = '<lists>';
671+
if ($line !== $expect) {
672+
throw new Exception("Expected '{$expect}', got {$line}.");
673+
}
674+
$mode = 'list1';
675+
break;
676+
case 'list1':
677+
$expect = '<list';
678+
if ($line !== $expect) {
679+
throw new Exception("Expected '{$expect}', got {$line}.");
680+
}
681+
$mode = 'list2';
682+
break;
683+
case 'list2':
684+
if (!preg_match('/^\s+path="/', $line)) {
685+
throw new Exception("Expected ' path=...', got {$line}.");
686+
}
687+
$mode = 'entry-or-end';
688+
break;
689+
}
690+
}
691+
if (!$done) {
692+
throw new Exception("Unexpected end of file.");
610693
}
611694

612695
return $map;
@@ -635,10 +718,3 @@ private function expandAllParentPaths($path, $include_self = false) {
635718
}
636719

637720
}
638-
639-
640-
641-
642-
643-
644-

‎src/applications/repository/worker/commitchangeparser/svn/__init__.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
phutil_require_module('phabricator', 'storage/qsprintf');
1414
phutil_require_module('phabricator', 'storage/queryfx');
1515

16+
phutil_require_module('phutil', 'filesystem');
17+
phutil_require_module('phutil', 'filesystem/linesofalargefile');
18+
phutil_require_module('phutil', 'filesystem/tempfile');
1619
phutil_require_module('phutil', 'future/exec');
1720
phutil_require_module('phutil', 'utils');
1821

0 commit comments

Comments
 (0)