Skip to content

Commit fe01949

Browse files
author
epriestley
committed
Add a Nuance GitHub repository source and basic polling
Summary: Ref T10537. Ref T10538. This calls GitHub, sorta? Test Plan: ``` $ ./bin/nuance import --source poem <cursor:events.repository> Polling GitHub Repository API endpoint "/repos/epriestley/poems/events". <cursor:events.repository> This key has 4,988 remaining API request(s), limit resets in 1,871 second(s). <cursor:events.repository> ETag for this request was ""4abdd3d66ad5ca38f5117b094e76f4ba"". array(4) { [0]=> array(7) { ["id"]=> string(10) "3733510485" ... ``` Reviewers: chad Reviewed By: chad Maniphest Tasks: T10537, T10538 Differential Revision: https://secure.phabricator.com/D15439
1 parent 2a3c3b2 commit fe01949

9 files changed

+325
-30
lines changed

‎src/__phutil_library_map__.php

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1421,6 +1421,8 @@
14211421
'NuanceController' => 'applications/nuance/controller/NuanceController.php',
14221422
'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php',
14231423
'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php',
1424+
'NuanceGitHubRepositoryImportCursor' => 'applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php',
1425+
'NuanceGitHubRepositorySourceDefinition' => 'applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php',
14241426
'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php',
14251427
'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php',
14261428
'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php',
@@ -5668,6 +5670,8 @@
56685670
'NuanceController' => 'PhabricatorController',
56695671
'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod',
56705672
'NuanceDAO' => 'PhabricatorLiskDAO',
5673+
'NuanceGitHubRepositoryImportCursor' => 'NuanceImportCursor',
5674+
'NuanceGitHubRepositorySourceDefinition' => 'NuanceSourceDefinition',
56715675
'NuanceImportCursor' => 'Phobject',
56725676
'NuanceImportCursorData' => 'NuanceDAO',
56735677
'NuanceImportCursorDataQuery' => 'NuanceQuery',

‎src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
<?php
2+
3+
final class NuanceGitHubRepositoryImportCursor
4+
extends NuanceImportCursor {
5+
6+
const CURSORTYPE = 'github.repository';
7+
8+
protected function shouldPullDataFromSource() {
9+
$now = PhabricatorTime::getNow();
10+
11+
// Respect GitHub's poll interval header. If we made a request recently,
12+
// don't make another one until we've waited long enough.
13+
$ttl = $this->getCursorProperty('github.poll.ttl');
14+
if ($ttl && ($ttl >= $now)) {
15+
$this->logInfo(
16+
pht(
17+
'Respecting "%s": waiting for %s second(s) to poll GitHub.',
18+
'X-Poll-Interval',
19+
new PhutilNumber(1 + ($ttl - $now))));
20+
21+
return false;
22+
}
23+
24+
// Respect GitHub's API rate limiting. If we've exceeded the rate limit,
25+
// wait until it resets to try again.
26+
$limit = $this->getCursorProperty('github.limit.ttl');
27+
if ($limit && ($limit >= $now)) {
28+
$this->logInfo(
29+
pht(
30+
'Respecting "%s": waiting for %s second(s) to poll GitHub.',
31+
'X-RateLimit-Reset',
32+
new PhutilNumber(1 + ($limit - $now))));
33+
return false;
34+
}
35+
36+
return true;
37+
}
38+
39+
protected function pullDataFromSource() {
40+
$source = $this->getSource();
41+
42+
$user = $source->getSourceProperty('github.user');
43+
$repository = $source->getSourceProperty('github.repository');
44+
$api_token = $source->getSourceProperty('github.token');
45+
46+
$uri = "/repos/{$user}/{$repository}/events";
47+
$data = array();
48+
49+
$future = id(new PhutilGitHubFuture())
50+
->setAccessToken($api_token)
51+
->setRawGitHubQuery($uri, $data);
52+
53+
$etag = $this->getCursorProperty('github.poll.etag');
54+
if ($etag) {
55+
$future->addHeader('If-None-Match', $etag);
56+
}
57+
58+
$this->logInfo(
59+
pht(
60+
'Polling GitHub Repository API endpoint "%s".',
61+
$uri));
62+
$response = $future->resolve();
63+
64+
// Do this first: if we hit the rate limit, we get a response but the
65+
// body isn't valid.
66+
$this->updateRateLimits($response);
67+
68+
// This means we hit a rate limit or a "Not Modified" because of the "ETag"
69+
// header. In either case, we should bail out.
70+
if ($response->getStatus()->isError()) {
71+
// TODO: Save cursor data!
72+
return false;
73+
}
74+
75+
$this->updateETag($response);
76+
77+
var_dump($response->getBody());
78+
}
79+
80+
private function updateRateLimits(PhutilGitHubResponse $response) {
81+
$remaining = $response->getHeaderValue('X-RateLimit-Remaining');
82+
$limit_reset = $response->getHeaderValue('X-RateLimit-Reset');
83+
$now = PhabricatorTime::getNow();
84+
85+
$limit_ttl = null;
86+
if (strlen($remaining)) {
87+
$remaining = (int)$remaining;
88+
if (!$remaining) {
89+
$limit_ttl = (int)$limit_reset;
90+
}
91+
}
92+
93+
$this->setCursorProperty('github.limit.ttl', $limit_ttl);
94+
95+
$this->logInfo(
96+
pht(
97+
'This key has %s remaining API request(s), '.
98+
'limit resets in %s second(s).',
99+
new PhutilNumber($remaining),
100+
new PhutilNumber($limit_reset - $now)));
101+
}
102+
103+
private function updateETag(PhutilGitHubResponse $response) {
104+
$etag = $response->getHeaderValue('ETag');
105+
106+
$this->setCursorProperty('github.poll.etag', $etag);
107+
108+
$this->logInfo(
109+
pht(
110+
'ETag for this request was "%s".',
111+
$etag));
112+
}
113+
114+
}

‎src/applications/nuance/cursor/NuanceImportCursor.php

Lines changed: 90 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,97 @@
22

33
abstract class NuanceImportCursor extends Phobject {
44

5+
private $cursorData;
6+
private $cursorKey;
7+
private $source;
8+
9+
abstract protected function shouldPullDataFromSource();
10+
abstract protected function pullDataFromSource();
11+
12+
final public function getCursorType() {
13+
return $this->getPhobjectClassConstant('CURSORTYPE', 32);
14+
}
15+
16+
public function setCursorData(NuanceImportCursorData $cursor_data) {
17+
$this->cursorData = $cursor_data;
18+
return $this;
19+
}
20+
21+
public function getCursorData() {
22+
return $this->cursorData;
23+
}
24+
25+
public function setSource($source) {
26+
$this->source = $source;
27+
return $this;
28+
}
29+
30+
public function getSource() {
31+
return $this->source;
32+
}
33+
34+
public function setCursorKey($cursor_key) {
35+
$this->cursorKey = $cursor_key;
36+
return $this;
37+
}
38+
39+
public function getCursorKey() {
40+
return $this->cursorKey;
41+
}
42+
543
final public function importFromSource() {
6-
// TODO: Perhaps, do something.
7-
return false;
44+
if (!$this->shouldPullDataFromSource()) {
45+
return false;
46+
}
47+
48+
$source = $this->getSource();
49+
$key = $this->getCursorKey();
50+
51+
$parts = array(
52+
'nsc',
53+
$source->getID(),
54+
PhabricatorHash::digestToLength($key, 20),
55+
);
56+
$lock_name = implode('.', $parts);
57+
58+
$lock = PhabricatorGlobalLock::newLock($lock_name);
59+
$lock->lock(1);
60+
61+
try {
62+
$more_data = $this->pullDataFromSource();
63+
} catch (Exception $ex) {
64+
$lock->unlock();
65+
throw $ex;
66+
}
67+
68+
$lock->unlock();
69+
70+
return $more_data;
71+
}
72+
73+
final public function newEmptyCursorData(NuanceSource $source) {
74+
return id(new NuanceImportCursorData())
75+
->setCursorKey($this->getCursorKey())
76+
->setCursorType($this->getCursorType())
77+
->setSourcePHID($source->getPHID());
78+
}
79+
80+
final protected function logInfo($message) {
81+
echo tsprintf(
82+
"<cursor:%s> %s\n",
83+
$this->getCursorKey(),
84+
$message);
85+
86+
return $this;
87+
}
88+
89+
final protected function getCursorProperty($key, $default = null) {
90+
return $this->getCursorData()->getCursorProperty($key, $default);
91+
}
92+
93+
final protected function setCursorProperty($key, $value) {
94+
$this->getCursorData()->setCursorProperty($key, $value);
95+
return $this;
896
}
997

1098
}

‎src/applications/nuance/management/NuanceManagementImportWorkflow.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ public function execute(PhutilArgumentParser $args) {
4040
$source->getName()));
4141
}
4242

43-
echo tsprintf(
44-
"%s\n",
45-
pht('OK, but actual importing is not implemented yet.'));
43+
foreach ($cursors as $cursor) {
44+
$cursor->importFromSource();
45+
}
4646

4747
return 0;
4848
}

‎src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?php
2+
3+
final class NuanceGitHubRepositorySourceDefinition
4+
extends NuanceSourceDefinition {
5+
6+
public function getName() {
7+
return pht('GitHub Repository');
8+
}
9+
10+
public function getSourceDescription() {
11+
return pht('Import issues and pull requests from a GitHub repository.');
12+
}
13+
14+
public function getSourceTypeConstant() {
15+
return 'github.repository';
16+
}
17+
18+
public function hasImportCursors() {
19+
return true;
20+
}
21+
22+
protected function newImportCursors() {
23+
return array(
24+
id(new NuanceGitHubRepositoryImportCursor())
25+
->setCursorKey('events.repository'),
26+
);
27+
}
28+
29+
}

‎src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,6 @@ public function getSourceViewActions(AphrontRequest $request) {
2626
return $actions;
2727
}
2828

29-
public function updateItems() {
30-
return null;
31-
}
32-
33-
public function renderView() {}
34-
35-
public function renderListView() {}
36-
37-
3829
public function handleActionRequest(AphrontRequest $request) {
3930
$viewer = $request->getViewer();
4031

‎src/applications/nuance/source/NuanceSourceDefinition.php

Lines changed: 66 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,66 @@ final public function getImportCursors() {
5353
pht('This source has no input cursors.'));
5454
}
5555

56-
return $this->newImportCursors();
56+
$source = $this->getSource();
57+
$cursors = $this->newImportCursors();
58+
59+
$data = id(new NuanceImportCursorDataQuery())
60+
->setViewer(PhabricatorUser::getOmnipotentUser())
61+
->withSourcePHIDs(array($source->getPHID()))
62+
->execute();
63+
$data = mpull($data, 'getCursorKey');
64+
65+
$map = array();
66+
foreach ($cursors as $cursor) {
67+
if (!($cursor instanceof NuanceImportCursor)) {
68+
throw new Exception(
69+
pht(
70+
'Source "%s" (of class "%s") returned an invalid value from '.
71+
'method "%s": all values must be objects of class "%s".',
72+
$this->getName(),
73+
get_class($this),
74+
'newImportCursors()',
75+
'NuanceImportCursor'));
76+
}
77+
78+
$key = $cursor->getCursorKey();
79+
if (!strlen($key)) {
80+
throw new Exception(
81+
pht(
82+
'Source "%s" (of class "%s") returned an import cursor with '.
83+
'a missing key from "%s". Each cursor must have a unique, '.
84+
'nonempty key.',
85+
$this->getName(),
86+
get_class($this),
87+
'newImportCursors()'));
88+
}
89+
90+
$other = idx($map, $key);
91+
if ($other) {
92+
throw new Exception(
93+
pht(
94+
'Source "%s" (of class "%s") returned two cursors from method '.
95+
'"%s" with the same key ("%s"). Each cursor must have a unique '.
96+
'key.',
97+
$this->getName(),
98+
get_class($this),
99+
'newImportCursors()',
100+
$key));
101+
}
102+
103+
$map[$key] = $cursor;
104+
105+
$cursor->setSource($source);
106+
107+
$cursor_data = idx($data, $key);
108+
if (!$cursor_data) {
109+
$cursor_data = $cursor->newEmptyCursorData($source);
110+
}
111+
112+
$cursor->setCursorData($cursor_data);
113+
}
114+
115+
return $cursors;
57116
}
58117

59118
protected function newImportCursors() {
@@ -79,21 +138,13 @@ abstract public function getSourceDescription();
79138
*/
80139
abstract public function getSourceTypeConstant();
81140

82-
/**
83-
* Code to create and update @{class:NuanceItem}s and
84-
* @{class:NuanceRequestor}s via daemons goes here.
85-
*
86-
* If that does not make sense for the @{class:NuanceSource} you are
87-
* defining, simply return null. For example,
88-
* @{class:NuancePhabricatorFormSourceDefinition} since these are one-way
89-
* contact forms.
90-
*/
91-
abstract public function updateItems();
92-
93-
abstract public function renderView();
94-
95-
abstract public function renderListView();
141+
public function renderView() {
142+
return null;
143+
}
96144

145+
public function renderListView() {
146+
return null;
147+
}
97148

98149
protected function newItemFromProperties(
99150
NuanceRequestor $requestor,

0 commit comments

Comments
 (0)