From 8d178b536dfa9a61f04947b3c7aa5e9bc7f90107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 11:55:48 +0100 Subject: [PATCH 1/9] Fetch all builds in one request --- jenkins_epo/extensions/jenkins.py | 38 +++++++++++++++---------------- jenkins_epo/jenkins.py | 32 +++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/jenkins_epo/extensions/jenkins.py b/jenkins_epo/extensions/jenkins.py index 2b2402f0..4bb6bc69 100644 --- a/jenkins_epo/extensions/jenkins.py +++ b/jenkins_epo/extensions/jenkins.py @@ -21,7 +21,7 @@ from jenkinsapi.custom_exceptions import UnknownJob from ..bot import Extension, Error, SkipHead -from ..jenkins import JENKINS, RESTClient +from ..jenkins import JENKINS from ..repository import Commit, CommitStatus from ..utils import match, switch_coro @@ -217,16 +217,24 @@ def process_job_specs(self): if update: yield JENKINS.update_job, spec + @asyncio.coroutine + def fetch_job(self, name): + if name in self.current.jobs: + return + try: + self.current.jobs[name] = yield from JENKINS.aget_job(name) + except UnknownJob: + pass + @asyncio.coroutine def run(self): logger.info("Fetching jobs from Jenkins.") - for name in self.current.job_specs: - if name in self.current.jobs: - continue - try: - self.current.jobs[name] = yield from JENKINS.aget_job(name) - except UnknownJob: - pass + loop = asyncio.get_event_loop() + tasks = [ + loop.create_task(self.fetch_job(name)) + for name in self.current.job_specs + ] + yield from asyncio.gather(*tasks) for action, spec in self.process_job_specs(): job = None @@ -324,19 +332,10 @@ def is_build_old(self, build): @asyncio.coroutine def poll_job(self, spec): job = self.current.jobs[spec.name] + payload = yield from job.fetch_builds() contextes = job.list_contexts(spec) - builds = reversed(sorted( - job.get_builds(), - key=lambda b: b.baseurl, - )) - for build in builds: - build._data = yield from ( - RESTClient(build.baseurl) - .aget(depth=build.depth) - ) - yield from switch_coro() + for build in job.process_builds(payload): if self.is_build_old(build): - logger.debug("Stop polling %s for older builds.", spec.name) break if not build._data['building']: @@ -372,6 +371,7 @@ def poll_job(self, spec): status = CommitStatus(context=job.name).from_build(build) logger.info("Queuing %s for cancel.", build) self.current.cancel_queue.append((commit, status)) + logger.debug("Polling %s done.", spec.name) @asyncio.coroutine def run(self): diff --git a/jenkins_epo/jenkins.py b/jenkins_epo/jenkins.py index 25ac479e..13dff494 100644 --- a/jenkins_epo/jenkins.py +++ b/jenkins_epo/jenkins.py @@ -21,7 +21,7 @@ import aiohttp from jenkinsapi.jenkinsbase import JenkinsBase -from jenkinsapi.build import Build +from jenkinsapi.build import Build as JenkinsBuild from jenkinsapi.jenkins import Jenkins, Requester from jenkins_yml import Job as JobSpec import requests @@ -188,6 +188,16 @@ def update_job(self, job_spec): JENKINS = LazyJenkins() +class Build(object): + def __init__(self, job, payload, api_instance): + self.job = job + self.payload = payload + self._instance = api_instance + + def __getattr__(self, name): + return getattr(self._instance, name) + + class Job(object): jobs_filter = parse_patterns(SETTINGS.JOBS) embedded_data_re = re.compile( @@ -285,6 +295,26 @@ def update_data_async(self): client = RESTClient(self._instance.baseurl) self._instance._data = yield from client.aget() + @asyncio.coroutine + def fetch_builds(self): + tree = "builds[" + ( + "actions[parameters[name,value]]," + "building,duration,number,result,timestamp,url" + ) + "]" + payload = yield from RESTClient(self.baseurl).aget(tree=tree) + return payload['builds'] + + def process_builds(self, payload): + payload = reversed(sorted(payload, key=lambda b: b['number'])) + for entry in payload: + api_instance = JenkinsBuild( + url=entry['url'], + buildno=entry['number'], + job=self._instance + ) + api_instance._data = entry + yield Build(self, entry, api_instance) + def get_builds(self): for number, url in self.get_build_dict().items(): yield Build(url, number, self._instance) From 2349e2d9aadd37cba10bea425cca56dcdc7e8741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 12:01:21 +0100 Subject: [PATCH 2/9] Use Jenkins python repr rather than json --- jenkins_epo/jenkins.py | 7 ++++--- tests/test_jenkins.py | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/jenkins_epo/jenkins.py b/jenkins_epo/jenkins.py index 13dff494..af3d50c2 100644 --- a/jenkins_epo/jenkins.py +++ b/jenkins_epo/jenkins.py @@ -13,6 +13,7 @@ # jenkins-epo. If not, see . +import ast import asyncio from datetime import datetime from itertools import product @@ -48,16 +49,16 @@ def __getattr__(self, name): def aget(self, **kw): session = aiohttp.ClientSession() - url = URL('%s/api/json' % (self.path)) + url = URL('%s/api/python' % (self.path)) if kw: url = url.with_query(**kw) logger.debug("GET %s", url) try: response = yield from session.get(url, timeout=10) - payload = yield from response.json() + payload = yield from response.read() finally: yield from session.close() - return payload + return ast.literal_eval(payload.decode('utf-8')) class VerboseRequester(Requester): diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index a51e323c..ef324ca5 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -15,9 +15,11 @@ def test_rest_client(mocker): session = ClientSession.return_value - response = Mock() + response = Mock(name='response') session.get = CoroutineMock(return_value=response) - response.json = CoroutineMock(return_value=dict(unittest=True)) + response.read = CoroutineMock( + return_value=repr(dict(unittest=True)).encode('utf-8') + ) payload = yield from client.aget(param=1) From fa56c702464ec5d4d34910456210e76be60450b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 12:11:28 +0100 Subject: [PATCH 3/9] Fetch config.xml async --- jenkins_epo/jenkins.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/jenkins_epo/jenkins.py b/jenkins_epo/jenkins.py index af3d50c2..fdf77994 100644 --- a/jenkins_epo/jenkins.py +++ b/jenkins_epo/jenkins.py @@ -47,9 +47,9 @@ def __call__(self, arg): def __getattr__(self, name): return self(name) - def aget(self, **kw): + def afetch(self, **kw): session = aiohttp.ClientSession() - url = URL('%s/api/python' % (self.path)) + url = URL(self.path) if kw: url = url.with_query(**kw) logger.debug("GET %s", url) @@ -58,7 +58,11 @@ def aget(self, **kw): payload = yield from response.read() finally: yield from session.close() - return ast.literal_eval(payload.decode('utf-8')) + return payload.decode('utf-8') + + def aget(self, **kw): + payload = yield from self.api.python.afetch(**kw) + return ast.literal_eval(payload) class VerboseRequester(Requester): @@ -133,8 +137,9 @@ def get_job(self, name): def aget_job(self, name): self.load() instance = self._instance.get_job(name) - data = yield from RESTClient(instance.baseurl).aget() - instance._data = data + client = RESTClient(instance.baseurl) + instance._data = yield from client.aget() + instance._config = yield from client('config.xml').afetch() return Job.factory(instance) DESCRIPTION_TMPL = """\ From eebc35c1207818e346a444890d74d5d3df78de9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 12:11:40 +0100 Subject: [PATCH 4/9] Set logging_id on subtasks --- jenkins_epo/extensions/jenkins.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins_epo/extensions/jenkins.py b/jenkins_epo/extensions/jenkins.py index 4bb6bc69..3eba64d3 100644 --- a/jenkins_epo/extensions/jenkins.py +++ b/jenkins_epo/extensions/jenkins.py @@ -219,6 +219,7 @@ def process_job_specs(self): @asyncio.coroutine def fetch_job(self, name): + asyncio.Task.current_task().logging_id = self.current.head.sha[:4] if name in self.current.jobs: return try: @@ -331,6 +332,7 @@ def is_build_old(self, build): @asyncio.coroutine def poll_job(self, spec): + asyncio.Task.current_task().logging_id = self.current.head.sha[:4] job = self.current.jobs[spec.name] payload = yield from job.fetch_builds() contextes = job.list_contexts(spec) From f4831dfb41900e1bed5cc32278b4c4da2c5ac03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 16:58:21 +0100 Subject: [PATCH 5/9] Move code to Build class --- jenkins_epo/extensions/jenkins.py | 77 ++-------- jenkins_epo/jenkins.py | 76 ++++++++-- tests/extensions/test_create_jobs.py | 2 + tests/extensions/test_poll.py | 205 +++++++-------------------- tests/test_jenkins.py | 105 ++++++++++++++ 5 files changed, 232 insertions(+), 233 deletions(-) diff --git a/jenkins_epo/extensions/jenkins.py b/jenkins_epo/extensions/jenkins.py index 3eba64d3..18b960bc 100644 --- a/jenkins_epo/extensions/jenkins.py +++ b/jenkins_epo/extensions/jenkins.py @@ -14,9 +14,7 @@ import asyncio from collections import OrderedDict -from datetime import datetime, timedelta import logging -import re from jenkinsapi.custom_exceptions import UnknownJob @@ -275,35 +273,6 @@ def process_error(self, spec, e): class PollExtension(JenkinsExtension): stage = '30' - ref_re = re.compile(r'.*origin/(?P.*)') - - def compute_build_ref_and_sha(self, job, build): - try: - jenkins_fullref = build.get_revision_branch()[0]['name'] - except IndexError: - for action in build._data['actions']: - if 'parameters' in action: - break - else: - return - for parameter in action['parameters']: - if parameter['name'] == job.revision_param: - break - else: - return - return ( - parameter['value'][len('refs/heads/'):], - self.current.last_commit.sha - ) - else: - match = self.ref_re.match(jenkins_fullref) - if not match: - return - return ( - match.group('ref'), - build.get_revision() - ) - def iter_preset_statuses(self, contextes, build): for context in contextes: default_status = CommitStatus( @@ -317,19 +286,6 @@ def iter_preset_statuses(self, contextes, build): status = status.from_build(build) yield status - def is_build_old(self, build): - now = datetime.now() - maxage = timedelta(hours=2) - seconds = build._data['timestamp'] / 1000. - build_date = datetime.fromtimestamp(seconds) - build_age = now - build_date - if build_date > now: - logger.warning( - "Build %s in the future. Is timezone correct?", build - ) - return False - return build_age > maxage - @asyncio.coroutine def poll_job(self, spec): asyncio.Task.current_task().logging_id = self.current.head.sha[:4] @@ -337,26 +293,19 @@ def poll_job(self, spec): payload = yield from job.fetch_builds() contextes = job.list_contexts(spec) for build in job.process_builds(payload): - if self.is_build_old(build): - break - - if not build._data['building']: + if not build.is_running: continue - try: - build_ref, build_sha = ( - self.compute_build_ref_and_sha(job, build) - ) - except TypeError: - logger.debug( - "Can't infer build ref and sha for %s.", build, - ) - continue + if build.is_outdated: + break - if build_ref != self.current.head.ref: + if build.ref != self.current.head.ref: continue - if build_sha == self.current.last_commit.sha: + try: + if build.sha == self.current.head.sha: + continue + except Exception: commit = self.current.last_commit preset_statuses = self.iter_preset_statuses( contextes, build, @@ -368,11 +317,11 @@ def poll_job(self, spec): commit.maybe_update_status(status) yield from switch_coro() continue - - commit = Commit(self.current.head.repository, build_sha) - status = CommitStatus(context=job.name).from_build(build) - logger.info("Queuing %s for cancel.", build) - self.current.cancel_queue.append((commit, status)) + else: + commit = Commit(self.current.head.repository, build.sha) + status = CommitStatus(context=job.name).from_build(build) + logger.info("Queuing %s for cancel.", build) + self.current.cancel_queue.append((commit, status)) logger.debug("Polling %s done.", spec.name) @asyncio.coroutine diff --git a/jenkins_epo/jenkins.py b/jenkins_epo/jenkins.py index fdf77994..e349c31d 100644 --- a/jenkins_epo/jenkins.py +++ b/jenkins_epo/jenkins.py @@ -15,7 +15,7 @@ import ast import asyncio -from datetime import datetime +from datetime import datetime, timedelta from itertools import product import logging import re @@ -47,6 +47,7 @@ def __call__(self, arg): def __getattr__(self, name): return self(name) + @retry def afetch(self, **kw): session = aiohttp.ClientSession() url = URL(self.path) @@ -195,14 +196,71 @@ def update_job(self, job_spec): class Build(object): - def __init__(self, job, payload, api_instance): + def __init__(self, job, payload, api_instance=None): self.job = job self.payload = payload self._instance = api_instance + self.params = self.process_params(payload) + + @staticmethod + def process_params(payload): + for action in payload.get('actions', []): + if 'parameters' in action: + break + else: + return {} + + return { + p['name']: p['value'] + for p in action['parameters'] + if 'value' in p + } def __getattr__(self, name): return getattr(self._instance, name) + def __str__(self): + return str(self._instance) + + @property + def is_outdated(self): + now = datetime.now() + maxage = timedelta(hours=2) + seconds = self.payload['timestamp'] / 1000. + build_date = datetime.fromtimestamp(seconds) + if build_date > now: + logger.warning( + "Build %s in the future. Is timezone correct?", self + ) + return False + build_age = now - build_date + return build_age > maxage + + @property + def is_running(self): + return self.payload['building'] + + _ref_re = re.compile(r'.*origin/(?P.*)') + + @property + def ref(self): + try: + fullref = self.payload['lastBuiltRevision']['branch']['name'] + except (TypeError, KeyError): + return self.params[self.job.revision_param][len('refs/heads/'):] + else: + match = self._ref_re.match(fullref) + if not match: + raise Exception("Unknown branch %s" % fullref) + return match.group('ref') + + @property + def sha(self): + try: + return self.payload['lastBuiltRevision']['branch']['SHA1'] + except (TypeError, KeyError): + raise Exception("No SHA1 yet.") + class Job(object): jobs_filter = parse_patterns(SETTINGS.JOBS) @@ -296,15 +354,13 @@ def node_param(self): break return self._node_param - @asyncio.coroutine - def update_data_async(self): - client = RESTClient(self._instance.baseurl) - self._instance._data = yield from client.aget() - @asyncio.coroutine def fetch_builds(self): tree = "builds[" + ( - "actions[parameters[name,value]]," + "actions[" + ( + "parameters[name,value]," + "lastBuiltRevision[branch[name,SHA1]]" + ) + "]," "building,duration,number,result,timestamp,url" ) + "]" payload = yield from RESTClient(self.baseurl).aget(tree=tree) @@ -321,10 +377,6 @@ def process_builds(self, payload): api_instance._data = entry yield Build(self, entry, api_instance) - def get_builds(self): - for number, url in self.get_build_dict().items(): - yield Build(url, number, self._instance) - class FreestyleJob(Job): def list_contexts(self, spec): diff --git a/tests/extensions/test_create_jobs.py b/tests/extensions/test_create_jobs.py index 42137eb5..0c76a723 100644 --- a/tests/extensions/test_create_jobs.py +++ b/tests/extensions/test_create_jobs.py @@ -226,6 +226,7 @@ def test_jenkins_create_success(mocker): ext = CreateJobsExtension('createjob', Mock()) ext.current = ext.bot.current + ext.current.head.sha = 'cafed0d0' ext.current.head.repository.jobs = {} ext.current.job_specs = dict(new=JobSpec('new', dict(periodic=True))) ext.current.jobs = {} @@ -254,6 +255,7 @@ def test_jenkins_fails_existing(mocker): ext = CreateJobsExtension('createjob', Mock()) ext.current = ext.bot.current ext.current.errors = [] + ext.current.head.sha = 'cafed0d0' ext.current.head.repository.jobs = {'job': Mock()} ext.current.job_specs = dict(job=JobSpec.factory('job', 'toto')) ext.current.jobs = {'job': Mock()} diff --git a/tests/extensions/test_poll.py b/tests/extensions/test_poll.py index 5420e05c..477ed37c 100644 --- a/tests/extensions/test_poll.py +++ b/tests/extensions/test_poll.py @@ -1,6 +1,5 @@ import asyncio from unittest.mock import Mock -from time import time from asynctest import CoroutineMock import pytest @@ -8,155 +7,100 @@ @pytest.mark.asyncio @asyncio.coroutine -def test_jenkins_skip_outdated(mocker): - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() +def test_jenkins_skip_outdated(): from jenkins_epo.extensions.jenkins import PollExtension ext = PollExtension('test', Mock()) ext.current = ext.bot.current + ext.current.head.sha = 'cafed0d0' ext.current.cancel_queue = [] ext.current.job_specs = {'job': Mock()} ext.current.job_specs['job'].name = 'job' ext.current.jobs = {} ext.current.jobs['job'] = job = Mock() - job.get_builds.return_value = builds = [Mock()] + job.fetch_builds = CoroutineMock() + job.process_builds.return_value = builds = [Mock()] build = builds[0] - aget.return_value = {'timestamp': (time() - 7 * 3600) * 1000} + build.is_outdated = True yield from ext.run() - assert not build.is_running.mock_calls + assert not builds[0].is_running.mock_calls assert 0 == len(ext.current.cancel_queue) @pytest.mark.asyncio @asyncio.coroutine -def test_jenkins_wrong_timezone(mocker): - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() +def test_jenkins_skip_build_not_running(): from jenkins_epo.extensions.jenkins import PollExtension ext = PollExtension('test', Mock()) ext.current = ext.bot.current + ext.current.head.sha = 'cafed0d0' ext.current.cancel_queue = [] ext.current.job_specs = {'job': Mock()} ext.current.job_specs['job'].name = 'job' ext.current.jobs = {} ext.current.jobs['job'] = job = Mock() - job.get_builds.return_value = [Mock()] - aget.return_value = { - 'timestamp': (time() + 2 * 3600) * 1000, 'building': False, - } - - yield from ext.run() - - assert 0 == len(ext.current.cancel_queue) - - -@pytest.mark.asyncio -@asyncio.coroutine -def test_jenkins_skip_build_not_running(mocker): - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() - from jenkins_epo.extensions.jenkins import PollExtension - - ext = PollExtension('test', Mock()) - ext.current = ext.bot.current - ext.current.cancel_queue = [] - ext.current.job_specs = {'job': Mock()} - ext.current.job_specs['job'].name = 'job' - ext.current.jobs = {} - ext.current.jobs['job'] = job = Mock() - job.get_builds.return_value = [Mock()] - aget.return_value = { - 'timestamp': (time() - 7 * 3600) * 1000, 'building': False, - } - - yield from ext.run() - - assert 0 == len(ext.current.cancel_queue) - - -@pytest.mark.asyncio -@asyncio.coroutine -def test_jenkins_skip_other_branch(mocker): - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() - from time import time - from jenkins_epo.extensions.jenkins import PollExtension - - ext = PollExtension('test', Mock()) - ext.current = ext.bot.current - ext.current.cancel_queue = [] - ext.current.job_specs = {'job': Mock()} - ext.current.job_specs['job'].name = 'job' - ext.current.jobs = {} - ext.current.jobs['job'] = job = Mock() - job.get_builds.return_value = builds = [Mock()] - ext.current.head.ref = 'branch' + job.fetch_builds = CoroutineMock() + job.process_builds.return_value = builds = [Mock()] build = builds[0] - aget.return_value = {'timestamp': time() * 1000, 'building': True} - build.get_revision_branch.return_value = [ - {'name': 'refs/remote/origin/other'} - ] + build.is_outdated = False + build.is_running = False yield from ext.run() - assert build.get_revision_branch.mock_calls assert 0 == len(ext.current.cancel_queue) @pytest.mark.asyncio @asyncio.coroutine -def test_jenkins_skip_unknown_branch(mocker): - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() - from time import time +def test_jenkins_skip_other_branch(): from jenkins_epo.extensions.jenkins import PollExtension ext = PollExtension('test', Mock()) ext.current = ext.bot.current ext.current.cancel_queue = [] + ext.current.head.sha = 'cafed0d0' ext.current.head.ref = 'branch' ext.current.job_specs = {'job': Mock()} ext.current.job_specs['job'].name = 'job' ext.current.jobs = {} ext.current.jobs['job'] = job = Mock() - job.get_builds.return_value = builds = [Mock()] + job.fetch_builds = CoroutineMock() + job.process_builds.return_value = builds = [Mock()] build = builds[0] - aget.return_value = {'timestamp': time() * 1000, 'building': True} - build.get_revision_branch.return_value = [{'name': 'otherremote/other'}] + build.is_outdated = False + build.is_running = True + build.ref = 'otherbranch' yield from ext.run() - assert build.get_revision_branch.mock_calls assert 0 == len(ext.current.cancel_queue) @pytest.mark.asyncio @asyncio.coroutine def test_jenkins_skip_current_sha(mocker): - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() - from time import time from jenkins_epo.extensions.jenkins import PollExtension ext = PollExtension('test', Mock()) ext.current = ext.bot.current ext.current.cancel_queue = [] ext.current.head.ref = 'branch' - ext.current.last_commit.sha = 'bab1' + ext.current.head.sha = 'bab1' ext.current.job_specs = {'job': Mock()} ext.current.job_specs['job'].name = 'job' ext.current.jobs = {} ext.current.jobs['job'] = job = Mock() job.list_contexts.return_value = [] - job.get_builds.return_value = builds = [Mock()] + job.fetch_builds = CoroutineMock() + job.process_builds.return_value = builds = [Mock()] build = builds[0] - aget.return_value = {'timestamp': time() * 1000, 'building': True} - build.get_revision_branch.return_value = [{'name': 'origin/branch'}] - build.get_revision.return_value = 'bab1' + build.is_outdated = False + build.is_running = True + build.ref = 'branch' + build.sha = ext.current.head.sha yield from ext.run() @@ -166,37 +110,31 @@ def test_jenkins_skip_current_sha(mocker): @pytest.mark.asyncio @asyncio.coroutine def test_jenkins_preset_status_cloning(mocker): - # When Jenkins is cloning, the build is real, we preset status with latest - # sha. - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() - from time import time + # When Jenkins is cloning, the build is real but no status is reported, we + # preset status on latest sha. from jenkins_epo.extensions.jenkins import PollExtension - ext = PollExtension('test', Mock()) + ext = PollExtension('test', Mock(name='bot')) ext.current = ext.bot.current ext.current.cancel_queue = [] ext.current.head.ref = 'branch' - ext.current.last_commit.sha = 'bab1' + ext.current.head.sha = 'bab1' ext.current.statuses = {} ext.current.job_specs = {'job': Mock()} ext.current.job_specs['job'].name = 'job' ext.current.jobs = {} ext.current.jobs['job'] = job = Mock() job.list_contexts.return_value = ['job'] - job.get_builds.return_value = builds = [Mock()] - job.revision_param = 'R' + job.fetch_builds = CoroutineMock() + job.process_builds.return_value = builds = [Mock(spec=[ + '_data', 'is_outdated', 'is_running', 'ref', 'baseurl', 'get_status', + ])] build = builds[0] - aget.return_value = { - 'timestamp': time() * 1000, 'building': True, - 'actions': [ - {'parameters': [{ - 'name': job.revision_param, - 'value': 'refs/heads/branch', - }]}, - ], - } - build.get_revision_branch.return_value = [] + build.is_outdated = False + build.is_running = True + build.ref = 'branch' + build.get_status.return_value = None + build._data = dict(displayName='#1', duration=1, url='url://') yield from ext.run() @@ -204,76 +142,29 @@ def test_jenkins_preset_status_cloning(mocker): assert ext.current.last_commit.maybe_update_status.mock_calls -@pytest.mark.asyncio -@asyncio.coroutine -def test_jenkins_preset_status_fail(mocker): - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() - from time import time - from jenkins_epo.extensions.jenkins import PollExtension - - ext = PollExtension('test', Mock()) - ext.current = ext.bot.current - ext.current.cancel_queue = [] - ext.current.head.ref = 'branch' - ext.current.last_commit.sha = 'bab1' - ext.current.statuses = {} - ext.current.job_specs = {'job': Mock()} - ext.current.job_specs['job'].name = 'job' - ext.current.jobs = {} - ext.current.jobs['job'] = job = Mock() - job.list_contexts.return_value = ['job'] - job.get_builds.return_value = builds = [Mock()] - job.revision_param = 'R' - build = builds[0] - build.get_revision_branch.return_value = [] - # Build with no parameters - aget.return_value = { - 'timestamp': time() * 1000, 'building': True, - 'actions': [], - } - - yield from ext.run() - - assert 0 == len(ext.current.cancel_queue) - assert not ext.current.last_commit.maybe_update_status.mock_calls - - # Build with no revision param - ext.current.last_commit.maybe_update_status.reset_mock() - aget.return_value['actions'].append({'parameters': []}) - - yield from ext.run() - - assert 0 == len(ext.current.cancel_queue) - assert not ext.current.last_commit.maybe_update_status.mock_calls - - @pytest.mark.asyncio @asyncio.coroutine def test_jenkins_cancel(mocker): - RESTClient = mocker.patch('jenkins_epo.extensions.jenkins.RESTClient') - RESTClient().aget = aget = CoroutineMock() - from time import time from jenkins_epo.extensions.jenkins import PollExtension ext = PollExtension('test', Mock()) ext.current = ext.bot.current ext.current.cancel_queue = [] ext.current.head.ref = 'branch' - ext.current.last_commit.sha = 'bab1' + ext.current.head.sha = 'bab1' ext.current.job_specs = {'job': Mock()} ext.current.job_specs['job'].name = 'job' ext.current.jobs = {} ext.current.jobs['job'] = job = Mock() - job.get_builds.return_value = builds = [Mock()] + job.fetch_builds = CoroutineMock() + job.process_builds.return_value = builds = [Mock()] build = builds[0] - aget.return_value = { - 'url': 'jenkins://running/1', - 'timestamp': time() * 1000, - 'building': True, - } - build.get_revision_branch.return_value = [{'name': 'origin/branch'}] - build.get_revision.return_value = '01d' + build.is_outdated = False + build.is_running = True + build.ref = 'branch' + build.sha = '01d' + build.get_status.return_value = None + build._data = dict(displayName='#1', duration=1, url='url://') yield from ext.run() diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index ef324ca5..56b98dd2 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -1,5 +1,6 @@ import asyncio from asynctest import patch, CoroutineMock, Mock +from time import time import pytest @@ -36,6 +37,110 @@ def test_lazy_load(mocker): assert JENKINS._instance +@pytest.mark.asyncio +@asyncio.coroutine +def test_fetch_builds(mocker): + RESTClient = mocker.patch('jenkins_epo.jenkins.RESTClient') + RESTClient().aget = aget = CoroutineMock(return_value=dict(builds=[])) + from jenkins_epo.jenkins import Job + + api_instance = Mock(_data=dict()) + api_instance.name = 'freestyle' + xml = api_instance._get_config_element_tree.return_value + xml.findall.return_value = [] + xml.find.return_value = None + + job = Job(api_instance) + yield from job.fetch_builds() + + assert aget.mock_calls + + +def test_process_builds(): + from jenkins_epo.jenkins import Job + + api_instance = Mock(_data=dict()) + api_instance.name = 'freestyle' + xml = api_instance._get_config_element_tree.return_value + xml.findall.return_value = [] + xml.find.return_value = None + + job = Job(api_instance) + + builds = list(job.process_builds([ + {'number': 1, 'url': 'url://'}, + {'number': 2, 'url': 'url://'}, + ])) + + assert 2 == len(builds) + assert 2 == builds[0].buildno + assert 1 == builds[1].buildno + + +def test_build_props(): + from jenkins_epo.jenkins import Build + + build = Build(job=Mock(), payload={ + 'timestamp': 1000 * (time() - 3600 * 4), + 'building': False, + }, api_instance=Mock()) + + assert build.is_outdated + assert not build.is_running + assert str(build) + + with pytest.raises(Exception): + build.sha + + build.payload['lastBuiltRevision'] = {'branch': {'SHA1': 'cafed0d0'}} + assert build.sha == 'cafed0d0' + + +def test_build_ref(): + from jenkins_epo.jenkins import Build + + build = Build(job=Mock(), payload={}, api_instance=Mock()) + + with pytest.raises(Exception): + build.ref + + build.job.revision_param = 'R' + build.params['R'] = 'refs/heads/master' + + assert 'master' == build.ref + + build.payload['lastBuiltRevision'] = { + 'branch': {'name': 'otherremote/master'} + } + + with pytest.raises(Exception): + build.ref + + build.payload['lastBuiltRevision'] = { + 'branch': {'name': 'refs/remote/origin/master'} + } + + assert 'master' == build.ref + + +def test_build_params(): + from jenkins_epo.jenkins import Build + + assert 0 == len(Build.process_params({})) + assert 0 == len(Build.process_params({'actions': [{'parameters': []}]})) + assert 0 == len(Build.process_params({ + 'actions': [{'parameters': [{'name': 'value'}]}] + })) + + +def test_build_future(): + from jenkins_epo.jenkins import Build + + build = Build(job=Mock(), payload={'timestamp': 1000 * (time() + 300)}) + + assert not build.is_outdated + + def test_freestyle_build(SETTINGS): from jenkins_epo.jenkins import FreestyleJob From 5deabdc30dac50a8843dad8d4a8a3086733d703a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 19:08:02 +0100 Subject: [PATCH 6/9] Test unknown Jenkins status --- tests/test_repositories.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_repositories.py b/tests/test_repositories.py index 347d8653..7dd14b07 100644 --- a/tests/test_repositories.py +++ b/tests/test_repositories.py @@ -469,6 +469,10 @@ def test_commit_status_from_build(): new_status = status.from_build(build) assert 'failure' == new_status['state'] + build.get_status.return_value = 'UNKNOWN' + new_status = status.from_build(build) + assert status == new_status + @patch('jenkins_epo.repository.cached_request') def test_commit_date(cached_request): From 126a1af06b6e1733de11b93fd0322c5c290423d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 20:11:44 +0100 Subject: [PATCH 7/9] Log traceback if VERBOSE --- jenkins_epo/workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins_epo/workers.py b/jenkins_epo/workers.py index 81a9942d..8cc2d607 100644 --- a/jenkins_epo/workers.py +++ b/jenkins_epo/workers.py @@ -75,7 +75,7 @@ def worker(self, id_): except CancelledError: logger.warn("Cancel of %s", item) except Exception as e: - if SETTINGS.DEBUG: + if SETTINGS.VERBOSE: logger.exception("Failed to process %s: %s", item, e) else: logger.error("Failed to process %s: %s", item, e) From 31bdb78ef71e5e5ba0a9e177743a49d013391f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 20:16:54 +0100 Subject: [PATCH 8/9] Avoid wrong GET log --- jenkins_epo/jenkins.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins_epo/jenkins.py b/jenkins_epo/jenkins.py index e349c31d..dab259d7 100644 --- a/jenkins_epo/jenkins.py +++ b/jenkins_epo/jenkins.py @@ -116,7 +116,6 @@ def load(self): @retry def is_queue_empty(self): - logging.debug("GET %s queue.", SETTINGS.JENKINS_URL) queue = self.get_queue() queue.poll() data = queue._data From c90e639d22cf2c0f0f70531c36c78583755ac491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20BERSAC=20=28bersace=29?= Date: Mon, 20 Feb 2017 20:19:01 +0100 Subject: [PATCH 9/9] Switch coro between build triggers --- jenkins_epo/extensions/jenkins.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins_epo/extensions/jenkins.py b/jenkins_epo/extensions/jenkins.py index 18b960bc..ee76fb1c 100644 --- a/jenkins_epo/extensions/jenkins.py +++ b/jenkins_epo/extensions/jenkins.py @@ -77,6 +77,7 @@ def run(self): rebuild_failed=self.current.rebuild_failed ) queue_empty = JENKINS.is_queue_empty() + yield from switch_coro() toqueue_contexts = [] for context in not_built: logger.debug("Computing new status for %s.", spec) @@ -99,6 +100,7 @@ def run(self): target_url=job.baseurl, ) ) + yield from switch_coro() def status_for_new_context(self, job, context, queue_empty): new_status = CommitStatus(target_url=job.baseurl, context=context)