Skip to content

Commit 3047354

Browse files
author
epriestley
committedJan 28, 2016
Add a basic pull event log for debugging repository cloning
Summary: Ref T10228. This is currently quite limited: - No UI. - No SSH support. My primary goal is to debug the issue in T10228. In the long run we can expand this to be a bit fancier. Test Plan: Made various valid and invalid clones, got sucess responses and not-so-successful responses, viewed the log table for general corresponding messages and broad sanity. Ran GC via `bin/phd debug trigger`, no issues. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10228 Differential Revision: https://secure.phabricator.com/D15127
1 parent c00cd5c commit 3047354

7 files changed

+371
-0
lines changed
 

‎resources/sql/autopatches/20160128.repo.1.pull.sql

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
CREATE TABLE {$NAMESPACE}_repository.repository_pullevent (
2+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3+
phid VARBINARY(64) NOT NULL,
4+
repositoryPHID VARBINARY(64),
5+
epoch INT UNSIGNED NOT NULL,
6+
pullerPHID VARBINARY(64),
7+
remoteAddress INT UNSIGNED,
8+
remoteProtocol VARCHAR(32) COLLATE {$COLLATE_TEXT},
9+
resultType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
10+
resultCode INT UNSIGNED NOT NULL,
11+
properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT},
12+
KEY `key_repository` (repositoryPHID),
13+
KEY `key_epoch` (epoch)
14+
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

‎src/__phutil_library_map__.php

+11
Original file line numberDiff line numberDiff line change
@@ -688,6 +688,7 @@
688688
'DiffusionPreCommitRefRepositoryHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefRepositoryHeraldField.php',
689689
'DiffusionPreCommitRefRepositoryProjectsHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefRepositoryProjectsHeraldField.php',
690690
'DiffusionPreCommitRefTypeHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefTypeHeraldField.php',
691+
'DiffusionPullEventGarbageCollector' => 'applications/diffusion/DiffusionPullEventGarbageCollector.php',
691692
'DiffusionPushCapability' => 'applications/diffusion/capability/DiffusionPushCapability.php',
692693
'DiffusionPushEventViewController' => 'applications/diffusion/controller/DiffusionPushEventViewController.php',
693694
'DiffusionPushLogController' => 'applications/diffusion/controller/DiffusionPushLogController.php',
@@ -3028,6 +3029,9 @@
30283029
'PhabricatorRepositoryMirrorQuery' => 'applications/repository/query/PhabricatorRepositoryMirrorQuery.php',
30293030
'PhabricatorRepositoryParsedChange' => 'applications/repository/data/PhabricatorRepositoryParsedChange.php',
30303031
'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php',
3032+
'PhabricatorRepositoryPullEvent' => 'applications/repository/storage/PhabricatorRepositoryPullEvent.php',
3033+
'PhabricatorRepositoryPullEventPHIDType' => 'applications/repository/phid/PhabricatorRepositoryPullEventPHIDType.php',
3034+
'PhabricatorRepositoryPullEventQuery' => 'applications/repository/query/PhabricatorRepositoryPullEventQuery.php',
30313035
'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php',
30323036
'PhabricatorRepositoryPushEvent' => 'applications/repository/storage/PhabricatorRepositoryPushEvent.php',
30333037
'PhabricatorRepositoryPushEventPHIDType' => 'applications/repository/phid/PhabricatorRepositoryPushEventPHIDType.php',
@@ -4708,6 +4712,7 @@
47084712
'DiffusionPreCommitRefRepositoryHeraldField' => 'DiffusionPreCommitRefHeraldField',
47094713
'DiffusionPreCommitRefRepositoryProjectsHeraldField' => 'DiffusionPreCommitRefHeraldField',
47104714
'DiffusionPreCommitRefTypeHeraldField' => 'DiffusionPreCommitRefHeraldField',
4715+
'DiffusionPullEventGarbageCollector' => 'PhabricatorGarbageCollector',
47114716
'DiffusionPushCapability' => 'PhabricatorPolicyCapability',
47124717
'DiffusionPushEventViewController' => 'DiffusionPushLogController',
47134718
'DiffusionPushLogController' => 'DiffusionController',
@@ -7476,6 +7481,12 @@
74767481
'PhabricatorRepositoryMirrorQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
74777482
'PhabricatorRepositoryParsedChange' => 'Phobject',
74787483
'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine',
7484+
'PhabricatorRepositoryPullEvent' => array(
7485+
'PhabricatorRepositoryDAO',
7486+
'PhabricatorPolicyInterface',
7487+
),
7488+
'PhabricatorRepositoryPullEventPHIDType' => 'PhabricatorPHIDType',
7489+
'PhabricatorRepositoryPullEventQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
74797490
'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
74807491
'PhabricatorRepositoryPushEvent' => array(
74817492
'PhabricatorRepositoryDAO',

‎src/applications/diffusion/DiffusionPullEventGarbageCollector.php

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?php
2+
3+
final class DiffusionPullEventGarbageCollector
4+
extends PhabricatorGarbageCollector {
5+
6+
const COLLECTORCONST = 'diffusion.pull';
7+
8+
public function getCollectorName() {
9+
return pht('Repository Pull Events');
10+
}
11+
12+
public function getDefaultRetentionPolicy() {
13+
return phutil_units('30 days in seconds');
14+
}
15+
16+
protected function collectGarbage() {
17+
$table = new PhabricatorRepositoryPullEvent();
18+
$conn_w = $table->establishConnection('w');
19+
20+
queryfx(
21+
$conn_w,
22+
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
23+
$table->getTableName(),
24+
$this->getGarbageEpoch());
25+
26+
return ($conn_w->getAffectedRows() == 100);
27+
}
28+
29+
}

‎src/applications/diffusion/controller/DiffusionServeController.php

+94
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,27 @@
22

33
final class DiffusionServeController extends DiffusionController {
44

5+
private $serviceViewer;
6+
private $serviceRepository;
7+
8+
public function setServiceViewer(PhabricatorUser $viewer) {
9+
$this->serviceViewer = $viewer;
10+
return $this;
11+
}
12+
13+
public function getServiceViewer() {
14+
return $this->serviceViewer;
15+
}
16+
17+
public function setServiceRepository(PhabricatorRepository $repository) {
18+
$this->serviceRepository = $repository;
19+
return $this;
20+
}
21+
22+
public function getServiceRepository() {
23+
return $this->serviceRepository;
24+
}
25+
526
public function isVCSRequest(AphrontRequest $request) {
627
$identifier = $this->getRepositoryIdentifierFromRequest($request);
728
if ($identifier === null) {
@@ -45,6 +66,75 @@ public function isVCSRequest(AphrontRequest $request) {
4566
}
4667

4768
public function handleRequest(AphrontRequest $request) {
69+
$service_exception = null;
70+
$response = null;
71+
72+
try {
73+
$response = $this->serveRequest($request);
74+
} catch (Exception $ex) {
75+
$service_exception = $ex;
76+
}
77+
78+
try {
79+
$remote_addr = $request->getRemoteAddr();
80+
$remote_addr = ip2long($remote_addr);
81+
82+
$pull_event = id(new PhabricatorRepositoryPullEvent())
83+
->setEpoch(PhabricatorTime::getNow())
84+
->setRemoteAddress($remote_addr)
85+
->setRemoteProtocol('http');
86+
87+
if ($response) {
88+
$pull_event
89+
->setResultType('wild')
90+
->setResultCode($response->getHTTPResponseCode());
91+
92+
if ($response instanceof PhabricatorVCSResponse) {
93+
$pull_event->setProperties(
94+
array(
95+
'response.message' => $response->getMessage(),
96+
));
97+
}
98+
} else {
99+
$pull_event
100+
->setResultType('exception')
101+
->setResultCode(500)
102+
->setProperties(
103+
array(
104+
'exception.class' => $ex->getClass(),
105+
'exception.message' => $ex->getMessage(),
106+
));
107+
}
108+
109+
$viewer = $this->getServiceViewer();
110+
if ($viewer) {
111+
$pull_event->setPullerPHID($viewer->getPHID());
112+
}
113+
114+
$repository = $this->getServiceRepository();
115+
if ($repository) {
116+
$pull_event->setRepositoryPHID($repository->getPHID());
117+
}
118+
119+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
120+
$pull_event->save();
121+
unset($unguarded);
122+
123+
} catch (Exception $ex) {
124+
if ($service_exception) {
125+
throw $service_exception;
126+
}
127+
throw $ex;
128+
}
129+
130+
if ($service_exception) {
131+
throw $service_exception;
132+
}
133+
134+
return $response;
135+
}
136+
137+
private function serveRequest(AphrontRequest $request) {
48138
$identifier = $this->getRepositoryIdentifierFromRequest($request);
49139

50140
// If authentication credentials have been provided, try to find a user
@@ -65,6 +155,8 @@ public function handleRequest(AphrontRequest $request) {
65155
$viewer = new PhabricatorUser();
66156
}
67157

158+
$this->setServiceViewer($viewer);
159+
68160
$allow_public = PhabricatorEnv::getEnvConfig('policy.allow-public');
69161
$allow_auth = PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth');
70162
if (!$allow_public) {
@@ -111,6 +203,8 @@ public function handleRequest(AphrontRequest $request) {
111203
}
112204
}
113205

206+
$this->setServiceRepository($repository);
207+
114208
if (!$repository->isTracked()) {
115209
return new PhabricatorVCSResponse(
116210
403,

‎src/applications/repository/phid/PhabricatorRepositoryPullEventPHIDType.php

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<?php
2+
3+
final class PhabricatorRepositoryPullEventPHIDType extends PhabricatorPHIDType {
4+
5+
const TYPECONST = 'PULE';
6+
7+
public function getTypeName() {
8+
return pht('Pull Event');
9+
}
10+
11+
public function newObject() {
12+
return new PhabricatorRepositoryPullEvent();
13+
}
14+
15+
public function getPHIDTypeApplicationClass() {
16+
return 'PhabricatorDiffusionApplication';
17+
}
18+
19+
protected function buildQueryForObjects(
20+
PhabricatorObjectQuery $query,
21+
array $phids) {
22+
23+
return id(new PhabricatorRepositoryPullEventQuery())
24+
->withPHIDs($phids);
25+
}
26+
27+
public function loadHandles(
28+
PhabricatorHandleQuery $query,
29+
array $handles,
30+
array $objects) {
31+
32+
foreach ($handles as $phid => $handle) {
33+
$event = $objects[$phid];
34+
35+
$handle->setName(pht('Pull Event %d', $event->getID()));
36+
}
37+
}
38+
39+
}

‎src/applications/repository/query/PhabricatorRepositoryPullEventQuery.php

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
<?php
2+
3+
final class PhabricatorRepositoryPullEventQuery
4+
extends PhabricatorCursorPagedPolicyAwareQuery {
5+
6+
private $ids;
7+
private $phids;
8+
private $repositoryPHIDs;
9+
private $pullerPHIDs;
10+
11+
public function withIDs(array $ids) {
12+
$this->ids = $ids;
13+
return $this;
14+
}
15+
16+
public function withPHIDs(array $phids) {
17+
$this->phids = $phids;
18+
return $this;
19+
}
20+
21+
public function withRepositoryPHIDs(array $repository_phids) {
22+
$this->repositoryPHIDs = $repository_phids;
23+
return $this;
24+
}
25+
26+
public function withPullerPHIDs(array $puller_phids) {
27+
$this->pullerPHIDs = $puller_phids;
28+
return $this;
29+
}
30+
31+
public function newResultObject() {
32+
return new PhabricatorRepositoryPullEvent();
33+
}
34+
35+
protected function loadPage() {
36+
return $this->loadStandardPage($this->newResultObject());
37+
}
38+
39+
protected function willFilterPage(array $events) {
40+
$repository_phids = mpull($events, 'getRepositoryPHID');
41+
$repositories = id(new PhabricatorRepositoryQuery())
42+
->setViewer($this->getViewer())
43+
->withPHIDs($repository_phids)
44+
->execute();
45+
$repositories = mpull($repositories, null, 'getPHID');
46+
47+
foreach ($events as $key => $event) {
48+
$phid = $event->getRepositoryPHID();
49+
if (empty($repositories[$phid])) {
50+
unset($events[$key]);
51+
continue;
52+
}
53+
$event->attachRepository($repositories[$phid]);
54+
}
55+
56+
return $events;
57+
}
58+
59+
protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
60+
$where = parent::buildWhereClauseParts($conn);
61+
62+
if ($this->ids !== null) {
63+
$where[] = qsprintf(
64+
$conn,
65+
'id IN (%Ld)',
66+
$this->ids);
67+
}
68+
69+
if ($this->phids !== null) {
70+
$where[] = qsprintf(
71+
$conn,
72+
'phid IN (%Ls)',
73+
$this->phids);
74+
}
75+
76+
if ($this->repositoryPHIDs !== null) {
77+
$where[] = qsprintf(
78+
$conn,
79+
'repositoryPHID IN (%Ls)',
80+
$this->repositoryPHIDs);
81+
}
82+
83+
if ($this->pullerPHIDs !== null) {
84+
$where[] = qsprintf(
85+
$conn,
86+
'pullerPHID in (%Ls)',
87+
$this->pullerPHIDs);
88+
}
89+
90+
return $where;
91+
}
92+
93+
public function getQueryApplicationClass() {
94+
return 'PhabricatorDiffusionApplication';
95+
}
96+
97+
}

0 commit comments

Comments
 (0)
Failed to load comments.