Merge pull request #274 from novafloss/commits

Async commits

jenkins_epo/bot.py

@@ -22,7 +22,6 @@
 import yaml
 
 from .github import GITHUB
-from .repository import Commit
 from .settings import SETTINGS
 from .utils import Bunch, parse_datetime, match, parse_patterns
 
@@ -111,11 +110,12 @@ def workon(self, head):
     def run(self, head):
         self.workon(head)
 
-        sha = self.current.head.sha
-        payload = yield from self.current.head.repository.fetch_commit(sha)
-        self.current.last_commit = Commit(
-            self.current.head.repository, sha, payload,
+        logger.info("Listing commits from GitHub.")
+        payload = yield from self.current.head.fetch_commits()
+        self.current.commits = list(
+            self.current.repository.process_commits(payload)
         )
+        self.current.last_commit = self.current.commits[0]
 
         logger.info("Fetching latest job status on GitHub.")
         payload = yield from self.current.last_commit.fetch_statuses()

jenkins_epo/extensions/core.py

@@ -28,7 +28,7 @@
 from ..jenkins import Job
 from ..repository import Branch, CommitStatus
 from ..settings import SETTINGS
-from ..utils import deepupdate, match, parse_patterns
+from ..utils import deepupdate, log_context, match, parse_patterns
 
 
 logger = logging.getLogger(__name__)
@@ -38,35 +38,39 @@ class AutoCancelExtension(Extension):
     stage = '30'
 
     @asyncio.coroutine
-    def run(self):
+    def process_commit(self, commit):
+        log_context(self.current.head)
+        is_head = commit.sha == self.current.head.sha
         now = datetime.datetime.utcnow()
         max_age = datetime.timedelta(seconds=3600)
-        logger.info("Listing previous commits from GitHub.")
-        payload = self.current.head.fetch_previous_commits()
-        commits = self.current.repository.process_commits(payload)
-        head = True
-        for i, commit in enumerate(commits):
-            age = now - commit.date
-            if i > 0 and age > max_age:
-                continue
+        age = now - commit.date
+        if not is_head and age > max_age:
+            return
 
-            commit_payload = yield from commit.fetch_statuses()
-            statuses = commit.process_statuses(commit_payload)
-            for status in statuses.values():
-                status = CommitStatus(status)
-                if status.get('state') != 'pending':
-                    continue
+        commit_payload = yield from commit.fetch_statuses()
+        statuses = commit.process_statuses(commit_payload)
+        for status in statuses.values():
+            status = CommitStatus(status)
+            if status.get('state') != 'pending':
+                continue
 
-                if status.is_queueable:
-                    continue
+            if status.is_queueable:
+                continue
 
-                if head:
-                    self.current.poll_queue.append((commit, status))
-                else:
-                    logger.info("Queue cancel of %s on %s.", status, commit)
-                    self.current.cancel_queue.append((commit, status))
+            if is_head:
+                self.current.poll_queue.append((commit, status))
+            else:
+                logger.info("Queue cancel of %s on %s.", status, commit)
+                self.current.cancel_queue.append((commit, status))
 
-            head = False
+    @asyncio.coroutine
+    def run(self):
+        loop = asyncio.get_event_loop()
+        tasks = [
+            loop.create_task(self.process_commit(commit))
+            for commit in self.current.commits
+        ]
+        yield from asyncio.gather(*tasks)
 
 
 class HelpExtension(Extension):

jenkins_epo/extensions/jenkins.py

@@ -16,6 +16,7 @@
 from collections import OrderedDict
 import logging
 
+from aiohttp.errors import HttpProcessingError
 from jenkinsapi.custom_exceptions import UnknownJob
 
 from ..bot import Extension, Error, SkipHead
@@ -135,6 +136,12 @@ def poll_build(self, commit, status, cancel):
         logger.debug("Query Jenkins %s status for %s.", status, commit)
         try:
             build = yield from Build.from_url(status['target_url'])
+        except HttpProcessingError as e:
+            logger.warn(
+                "Failed to get %s: %s %s",
+                status['target_url'], e.code, e.message,
+            )
+            return
         except NotOnJenkins as e:
             logger.debug("%s not on this Jenkins", status['target_url'])
             return

jenkins_epo/repository.py

@@ -506,15 +506,16 @@ def __hash__(self):
     def url(self):
         return 'https://github.com/%s/tree/%s' % (self.repository, self.ref)
 
-    def fetch_previous_commits(self, last_date=None):
-        head = cached_request(
+    @asyncio.coroutine
+    def fetch_commits(self, last_date=None):
+        head = yield from cached_arequest(
             GITHUB.repos(self.repository).git.commits(self.sha)
         )
-        yield head
-        yield cached_request(
+        parent = yield from cached_arequest(
             GITHUB.repos(self.repository).git
             .commits(head['parents'][0]['sha'])
         )
+        return [head, parent]
 
     @asyncio.coroutine
     def fetch_comments(self):
@@ -567,14 +568,16 @@ def author(self):
     def url(self):
         return self.payload['html_url']
 
-    def fetch_previous_commits(self, last_date=None):
+    @asyncio.coroutine
+    def fetch_commits(self, last_date=None):
         logger.debug("Fetching previous commits.")
-        payload = cached_request(GITHUB.repos(self.repository).compare(
-            urlquote("%s...%s" % (
+        payload = yield from cached_arequest(
+            GITHUB.repos(self.repository)
+            .compare(urlquote("%s...%s" % (
                 self.payload['base']['label'],
                 self.payload['head']['label'],
-            ))
-        ))
+            )))
+        )
         return reversed(payload['commits'])
 
     @retry

tests/extensions/test_core.py

@@ -159,19 +159,17 @@ def test_autocancel():
     ext.current = Mock()
     ext.current.cancel_queue = cancel_queue = []
     ext.current.poll_queue = []
-    last_commit = Mock(
-        name='last', date=datetime.utcnow() - timedelta(seconds=50000)
+    ext.current.last_commit = last_commit = Mock(
+        name='last', date=datetime.utcnow() - timedelta(seconds=50000),
     )
-    ext.current.last_commit = last_commit
+    ext.current.head.sha = ext.current.last_commit.sha = 'cafed0d0'
     old_commit = Mock(
         name='old', date=datetime.utcnow() - timedelta(seconds=300)
     )
     outdated_commit = Mock(
         name='outdated', date=datetime.utcnow() - timedelta(seconds=4000)
     )
-    ext.current.repository.process_commits.return_value = [
-        last_commit, old_commit, outdated_commit,
-    ]
+    ext.current.commits = [last_commit, old_commit, outdated_commit]
 
     last_commit.fetch_statuses.return_value = []
     last_commit.process_statuses.return_value = last_statuses = {

tests/extensions/test_jenkins.py

@@ -145,6 +145,41 @@ def test_cancel_ignore_other(mocker):
     assert not commit.maybe_update_status.mock_calls
 
 
+@pytest.mark.asyncio
+@asyncio.coroutine
+def test_cancel_404(mocker):
+    JENKINS = mocker.patch('jenkins_epo.extensions.jenkins.JENKINS')
+    Build = mocker.patch('jenkins_epo.extensions.jenkins.Build')
+    from jenkins_epo.extensions.jenkins import (
+        CancellerExtension, CommitStatus, HttpProcessingError
+    )
+
+    JENKINS.baseurl = 'jenkins://'
+
+    commit = Mock()
+
+    ext = CancellerExtension('test', Mock())
+    ext.current = ext.bot.current
+    ext.current.head.sha = 'cafed0d0'
+    ext.current.poll_queue = []
+    ext.current.cancel_queue = [
+        (commit, CommitStatus(context='job', target_url='jenkins://job/1')),
+    ]
+    ext.current.SETTINGS.DRY_RUN = 0
+    ext.current.last_commit.fetch_statuses.return_value = []
+    ext.current.statuses = {}
+
+    Build.from_url = CoroutineMock(
+        side_effect=HttpProcessingError(code=404, message='Not Found')
+    )
+    build = Build.from_url.return_value
+
+    yield from ext.run()
+
+    assert not build.stop.mock_calls
+    assert not commit.maybe_update_status.mock_calls
+
+
 @pytest.mark.asyncio
 @asyncio.coroutine
 def test_cancel_build_running(mocker):

tests/test_bot.py

@@ -124,7 +124,6 @@ def test_parse_error():
 @asyncio.coroutine
 def test_run_extension(mocker):
     pkg_resources = mocker.patch('jenkins_epo.bot.pkg_resources')
-    mocker.patch('jenkins_epo.bot.Commit')
 
     from jenkins_epo.bot import Bot
 
@@ -139,12 +138,17 @@ def test_run_extension(mocker):
     bot = Bot()
     assert 'ext' in bot.extensions_map
 
-    pr = Mock()
+    pr = Mock(name='pr')
+    pr.fetch_commits = CoroutineMock()
+    commits = [Mock()]
+    pr.repository.process_commits.return_value = commits
     pr.fetch_comments = CoroutineMock(return_value=[])
-    pr.repository.fetch_commit.return_value = []
+    commits[0].fetch_statuses = CoroutineMock()
 
     yield from bot.run(pr)
 
+    assert pr.fetch_commits.mock_calls
+    assert pr.fetch_comments.mock_calls
     assert ext.begin.mock_calls
     assert ext.run.mock_calls
 
@@ -153,7 +157,6 @@ def test_run_extension(mocker):
 @asyncio.coroutine
 def test_begin_skip_head(mocker):
     pkg_resources = mocker.patch('jenkins_epo.bot.pkg_resources')
-    mocker.patch('jenkins_epo.bot.Commit')
 
     from jenkins_epo.bot import Bot, SkipHead
 
@@ -165,10 +168,12 @@ def test_begin_skip_head(mocker):
     ext.SETTINGS = {}
     ext.begin.side_effect = SkipHead()
 
-    pr = Mock()
-    pr.sha = 'cafed0d0'
+    pr = Mock(name='pr')
+    pr.fetch_commits = CoroutineMock()
+    commits = [Mock()]
+    pr.repository.process_commits.return_value = commits
     pr.fetch_comments = CoroutineMock(return_value=[])
-    pr.repository.fetch_commit.return_value = []
+    commits[0].fetch_statuses = CoroutineMock()
 
     yield from Bot().run(pr)
 
@@ -180,7 +185,6 @@ def test_begin_skip_head(mocker):
 @asyncio.coroutine
 def test_run_skip_head(mocker):
     pkg_resources = mocker.patch('jenkins_epo.bot.pkg_resources')
-    mocker.patch('jenkins_epo.bot.Commit')
 
     from jenkins_epo.bot import Bot, SkipHead
 
@@ -192,10 +196,12 @@ def test_run_skip_head(mocker):
     ext.SETTINGS = {}
     ext.run.side_effect = SkipHead()
 
-    pr = Mock()
-    pr.sha = 'cafed0d0'
+    pr = Mock(name='pr')
+    pr.fetch_commits = CoroutineMock()
+    commits = [Mock()]
+    pr.repository.process_commits.return_value = commits
     pr.fetch_comments = CoroutineMock(return_value=[])
-    pr.repository.fetch_commit.return_value = []
+    commits[0].fetch_statuses = CoroutineMock()
 
     yield from Bot().run(pr)
 

tests/test_head.py

@@ -155,17 +155,22 @@ def test_sort_heads():
     assert wanted == computed
 
 
-@patch('jenkins_epo.repository.cached_request')
-def test_branch_fetch_previous_commits(cached_request):
-    cached_request.side_effect = [
+@pytest.mark.asyncio
+@asyncio.coroutine
+def test_branch_fetch_commits(mocker):
+    cached_arequest = mocker.patch(
+        'jenkins_epo.repository.cached_arequest', CoroutineMock()
+    )
+    cached_arequest.side_effect = [
         dict(parents=[dict(sha='d0d0cafe')]),
         dict()
     ]
     from jenkins_epo.repository import Branch
 
     head = Branch(Mock(), dict(name='branch', commit=dict(sha='d0d0cafe')))
-    assert list(head.fetch_previous_commits())
-    assert cached_request.mock_calls
+    payload = yield from head.fetch_commits()
+    assert payload
+    assert cached_arequest.mock_calls
 
 
 @pytest.mark.asyncio
@@ -184,16 +189,20 @@ def test_branch_fetch_comments(mocker):
     assert arequest.mock_calls
 
 
-@patch('jenkins_epo.repository.cached_request')
-def test_pr_fetch_previous_commits(cached_request):
+@pytest.mark.asyncio
+@asyncio.coroutine
+def test_pr_fetch_commits(mocker):
+    cached_arequest = mocker.patch(
+        'jenkins_epo.repository.cached_arequest', CoroutineMock()
+    )
+    cached_arequest.return_value = dict(commits=['previous', 'last'])
     from jenkins_epo.repository import PullRequest
-    cached_request.return_value = dict(commits=['previous', 'last'])
     head = PullRequest(Mock(), dict(
         head=dict(ref='pr', sha='d0d0cafe', label='owner:pr'),
         base=dict(label='owner:base'),
     ))
-    commits = list(head.fetch_previous_commits())
-    assert ['last', 'previous'] == commits
+    commits = yield from head.fetch_commits()
+    assert ['last', 'previous'] == list(commits)
 
 
 @pytest.mark.asyncio