Skip to content

Commit ef8c43a

Browse files
author
vrana
committed
Simplify and optimize save_lint.php
Test Plan: Ran it. Reviewers: epriestley Reviewed By: epriestley CC: aran, Korvin Maniphest Tasks: T2038 Differential Revision: https://secure.phabricator.com/D3933
1 parent 5a58d16 commit ef8c43a

File tree

3 files changed

+247
-98
lines changed

3 files changed

+247
-98
lines changed

‎scripts/repository/save_lint.php‎

Lines changed: 48 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -3,101 +3,51 @@
33

44
require_once dirname(__FILE__).'/../__init_script__.php';
55

6-
if (function_exists('posix_isatty') && posix_isatty(STDIN)) {
7-
$command = 'xargs -0 arc lint --output json | '.__FILE__;
8-
echo "Usage: git ls-files -z | {$command}\n";
9-
echo "Usage: git diff --name-only -z | {$command}\n"; // TODO: Handle deletes.
10-
echo "Purpose: Save all lint errors to database.\n";
11-
exit(1);
12-
}
13-
14-
$working_copy = ArcanistWorkingCopyIdentity::newFromPath('.');
15-
$api = ArcanistRepositoryAPI::newAPIFromWorkingCopyIdentity($working_copy);
16-
$svn_root = id(new PhutilURI($api->getSourceControlPath()))->getPath();
17-
18-
$project_id = $working_copy->getProjectID();
19-
$project = id(new PhabricatorRepositoryArcanistProject())
20-
->loadOneWhere('name = %s', $project_id);
21-
if (!$project || !$project->getRepositoryID()) {
22-
throw new Exception("Couldn't find repository for {$project_id}.");
23-
}
24-
25-
$branch_name = $api->getBranchName();
26-
$branch = id(new PhabricatorRepositoryBranch())->loadOneWhere(
27-
'repositoryID = %d AND name = %s',
28-
$project->getRepositoryID(),
29-
$branch_name);
30-
if (!$branch) {
31-
$branch = id(new PhabricatorRepositoryBranch())
32-
->setRepositoryID($project->getRepositoryID())
33-
->setName($branch_name);
34-
}
35-
$branch->setLintCommit($api->getWorkingCopyRevision());
36-
$branch->save();
37-
$conn = $branch->establishConnection('w');
38-
39-
$inserts = array();
40-
41-
while ($json = fgets(STDIN)) {
42-
$paths = json_decode(rtrim($json, "\n"), true);
43-
if (!is_array($paths)) {
44-
throw new Exception("Invalid JSON: {$json}");
45-
}
46-
47-
if (!$paths) {
48-
continue;
49-
}
50-
51-
$conn->openTransaction();
52-
53-
foreach (array_chunk(array_keys($paths), 1024) as $some_paths) {
54-
$full_paths = array();
55-
foreach ($some_paths as $path) {
56-
$full_paths[] = $svn_root.'/'.$path;
57-
}
58-
queryfx(
59-
$conn,
60-
'DELETE FROM %T WHERE branchID = %d AND path IN (%Ls)',
61-
PhabricatorRepository::TABLE_LINTMESSAGE,
62-
$branch->getID(),
63-
$full_paths);
64-
}
65-
66-
foreach ($paths as $path => $messages) {
67-
// TODO: Handle multiple $json for a single path. Don't save duplicates.
68-
69-
foreach ($messages as $message) {
70-
$inserts[] = qsprintf(
71-
$conn,
72-
'(%d, %s, %d, %s, %s, %s, %s)',
73-
$branch->getID(),
74-
$svn_root.'/'.$path,
75-
idx($message, 'line', 0),
76-
idx($message, 'code', ''),
77-
idx($message, 'severity', ''),
78-
idx($message, 'name', ''),
79-
idx($message, 'description', ''));
80-
81-
if (count($inserts) >= 256) {
82-
save_lint_messages($conn, $inserts);
83-
$inserts = array();
84-
}
85-
}
86-
}
87-
88-
$conn->saveTransaction();
89-
}
90-
91-
save_lint_messages($conn, $inserts);
92-
93-
function save_lint_messages($conn, array $inserts) {
94-
if ($inserts) {
95-
queryfx(
96-
$conn,
97-
'INSERT INTO %T
98-
(branchID, path, line, code, severity, name, description)
99-
VALUES %Q',
100-
PhabricatorRepository::TABLE_LINTMESSAGE,
101-
implode(', ', $inserts));
102-
}
103-
}
6+
$synopsis = <<<EOT
7+
**save_lint.php**
8+
Discover lint problems and save them to database so that they can
9+
be displayed in Diffusion.
10+
11+
EOT;
12+
13+
$args = id(new PhutilArgumentParser($argv))
14+
->setTagline('save lint errors to database')
15+
->setSynopsis($synopsis)
16+
->parseStandardArguments()
17+
->parse(array(
18+
array(
19+
'name' => 'all',
20+
'help' =>
21+
"Discover problems in the whole repository instead of just changes ".
22+
"since the last run.",
23+
),
24+
array(
25+
'name' => 'arc',
26+
'param' => 'path',
27+
'default' => 'arc',
28+
'help' => "Path to Arcanist executable.",
29+
),
30+
array(
31+
'name' => 'severity',
32+
'param' => 'string',
33+
'default' => ArcanistLintSeverity::SEVERITY_ADVICE,
34+
'help' => "Minimum severity, one of ArcanistLintSeverity constants.",
35+
),
36+
array(
37+
'name' => 'chunk-size',
38+
'param' => 'number',
39+
'default' => 256,
40+
'help' => "Number of paths passed to `arc` at once.",
41+
),
42+
));
43+
44+
echo "Saving lint errors to database...\n";
45+
46+
$count = id(new DiffusionLintSaveRunner())
47+
->setAll($args->getArg('all', false))
48+
->setArc($args->getArg('arc'))
49+
->setSeverity($args->getArg('severity'))
50+
->setChunkSize($args->getArg('chunk-size'))
51+
->run('.');
52+
53+
echo "\nProcessed {$count} files.\n";

‎src/__phutil_library_map__.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,7 @@
362362
'DiffusionLastModifiedQuery' => 'applications/diffusion/query/lastmodified/DiffusionLastModifiedQuery.php',
363363
'DiffusionLintController' => 'applications/diffusion/controller/DiffusionLintController.php',
364364
'DiffusionLintDetailsController' => 'applications/diffusion/controller/DiffusionLintDetailsController.php',
365+
'DiffusionLintSaveRunner' => 'applications/diffusion/DiffusionLintSaveRunner.php',
365366
'DiffusionMercurialBranchQuery' => 'applications/diffusion/query/branch/DiffusionMercurialBranchQuery.php',
366367
'DiffusionMercurialBrowseQuery' => 'applications/diffusion/query/browse/DiffusionMercurialBrowseQuery.php',
367368
'DiffusionMercurialCommitParentsQuery' => 'applications/diffusion/query/parents/DiffusionMercurialCommitParentsQuery.php',

‎src/applications/diffusion/DiffusionLintSaveRunner.php‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
<?php
2+
3+
final class DiffusionLintSaveRunner {
4+
private $arc = 'arc';
5+
private $severity = ArcanistLintSeverity::SEVERITY_ADVICE;
6+
private $all = false;
7+
private $chunkSize = 256;
8+
9+
private $svnRoot;
10+
private $lintCommit;
11+
private $branch;
12+
private $conn;
13+
private $deletes = array();
14+
private $inserts = array();
15+
16+
17+
public function setArc($path) {
18+
$this->arc = $path;
19+
return $this;
20+
}
21+
22+
public function setSeverity($string) {
23+
$this->severity = $string;
24+
return $this;
25+
}
26+
27+
public function setAll($bool) {
28+
$this->all = $bool;
29+
return $this;
30+
}
31+
32+
public function setChunkSize($number) {
33+
$this->chunkSize = $number;
34+
return $this;
35+
}
36+
37+
38+
public function run($dir) {
39+
$working_copy = ArcanistWorkingCopyIdentity::newFromPath($dir);
40+
$api = ArcanistRepositoryAPI::newAPIFromWorkingCopyIdentity($working_copy);
41+
$this->svnRoot = id(new PhutilURI($api->getSourceControlPath()))->getPath();
42+
43+
$project_id = $working_copy->getProjectID();
44+
$project = id(new PhabricatorRepositoryArcanistProject())
45+
->loadOneWhere('name = %s', $project_id);
46+
if (!$project || !$project->getRepositoryID()) {
47+
throw new Exception("Couldn't find repository for {$project_id}.");
48+
}
49+
50+
$branch_name = $api->getBranchName();
51+
$this->branch = new PhabricatorRepositoryBranch();
52+
$this->conn = $this->branch->establishConnection('w');
53+
$this->branch = $this->branch->loadOneWhere(
54+
'repositoryID = %d AND name = %s',
55+
$project->getRepositoryID(),
56+
$branch_name);
57+
58+
$this->lintCommit = null;
59+
if (!$this->branch) {
60+
$this->branch = id(new PhabricatorRepositoryBranch())
61+
->setRepositoryID($project->getRepositoryID())
62+
->setName($branch_name)
63+
->save();
64+
} else if (!$this->all) {
65+
$this->lintCommit = $this->branch->getLintCommit();
66+
}
67+
68+
if ($this->lintCommit) {
69+
try {
70+
$all_files = $api->getChangedFiles($this->lintCommit);
71+
} catch (ArcanistCapabilityNotSupportedException $ex) {
72+
$this->lintCommit = null;
73+
}
74+
}
75+
76+
if (!$this->lintCommit) {
77+
$where = ($this->svnRoot
78+
? qsprintf($this->conn, 'AND path LIKE %>', $this->svnRoot.'/')
79+
: '');
80+
queryfx(
81+
$this->conn,
82+
'DELETE FROM %T WHERE branchID = %d %Q',
83+
PhabricatorRepository::TABLE_LINTMESSAGE,
84+
$this->branch->getID(),
85+
$where);
86+
$all_files = $api->getAllFiles();
87+
}
88+
89+
$this->deletes = array();
90+
$this->inserts = array();
91+
$count = 0;
92+
93+
$files = array();
94+
foreach ($all_files as $file => $val) {
95+
$count++;
96+
if (!$this->lintCommit) {
97+
$file = $val;
98+
} else {
99+
$this->deletes[] = $this->svnRoot.'/'.$file;
100+
if ($val & ArcanistRepositoryAPI::FLAG_DELETED) {
101+
continue;
102+
}
103+
}
104+
$files[$file] = $file;
105+
106+
if (count($files) >= $this->chunkSize) {
107+
$this->runArcLint($files);
108+
$files = array();
109+
}
110+
}
111+
112+
$this->runArcLint($files);
113+
$this->saveLintMessages();
114+
$this->branch->setLintCommit($api->getWorkingCopyRevision());
115+
$this->branch->save();
116+
117+
return $count;
118+
}
119+
120+
121+
private function runArcLint(array $files) {
122+
if (!$files) {
123+
return;
124+
}
125+
126+
echo '.';
127+
try {
128+
$future = new ExecFuture(
129+
'%C lint --severity %s --output json %Ls',
130+
$this->arc,
131+
$this->severity,
132+
$files);
133+
134+
foreach (new LinesOfALargeExecFuture($future) as $json) {
135+
$paths = json_decode($json, true);
136+
if (!is_array($paths)) {
137+
fprintf(STDERR, "Invalid JSON: {$json}\n");
138+
continue;
139+
}
140+
141+
foreach ($paths as $path => $messages) {
142+
if (!isset($files[$path])) {
143+
continue;
144+
}
145+
146+
foreach ($messages as $message) {
147+
$this->inserts[] = qsprintf(
148+
$this->conn,
149+
'(%d, %s, %d, %s, %s, %s, %s)',
150+
$this->branch->getID(),
151+
$this->svnRoot.'/'.$path,
152+
idx($message, 'line', 0),
153+
idx($message, 'code', ''),
154+
idx($message, 'severity', ''),
155+
idx($message, 'name', ''),
156+
idx($message, 'description', ''));
157+
}
158+
159+
if (count($this->deletes) >= 1024 || count($this->inserts) >= 256) {
160+
$this->saveLintMessages($this->branch);
161+
$this->deletes = array();
162+
$this->inserts = array();
163+
}
164+
}
165+
}
166+
167+
} catch (Exception $ex) {
168+
fprintf(STDERR, $ex->getMessage()."\n");
169+
}
170+
}
171+
172+
173+
private function saveLintMessages() {
174+
$this->conn->openTransaction();
175+
176+
foreach (array_chunk($this->deletes, 1024) as $paths) {
177+
queryfx(
178+
$this->conn,
179+
'DELETE FROM %T WHERE branchID = %d AND path IN (%Ls)',
180+
PhabricatorRepository::TABLE_LINTMESSAGE,
181+
$this->branch->getID(),
182+
$paths);
183+
}
184+
185+
foreach (array_chunk($this->inserts, 256) as $values) {
186+
queryfx(
187+
$this->conn,
188+
'INSERT INTO %T
189+
(branchID, path, line, code, severity, name, description)
190+
VALUES %Q',
191+
PhabricatorRepository::TABLE_LINTMESSAGE,
192+
implode(', ', $values));
193+
}
194+
195+
$this->conn->saveTransaction();
196+
}
197+
198+
}

0 commit comments

Comments
 (0)