From 74f10400de38eee6995402da8c57a74d14980068 Mon Sep 17 00:00:00 2001 From: Pawel Krawczyk <616047+kravietz@users.noreply.github.com> Date: Fri, 2 Aug 2019 10:35:33 +0100 Subject: [PATCH 1/8] Fix #4873 per latest Bitbucket API documentation * replace the username property with nickname (per the privacy changes in API) https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-entity_user * per Bitbucket documentation the project property in repository is not guaranteed to be always present which was also crashing my Buildbot https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-entity_repositoryRepository --- master/buildbot/www/hooks/bitbucketcloud.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/master/buildbot/www/hooks/bitbucketcloud.py b/master/buildbot/www/hooks/bitbucketcloud.py index 93827ec9ade9..3a13d4201eef 100644 --- a/master/buildbot/www/hooks/bitbucketcloud.py +++ b/master/buildbot/www/hooks/bitbucketcloud.py @@ -70,7 +70,10 @@ def _get_payload(self, request): def handle_repo_push(self, payload): changes = [] + if payload['repository'].get('project'): project = payload['repository']['project']['name'] + else: + project = 'none' repo_url = payload['repository']['links']['self']['href'] web_url = payload['repository']['links']['html']['href'] @@ -94,7 +97,7 @@ def handle_repo_push(self, payload): 'revlink': '{}/commits/{}'.format(web_url, commit_hash), 'repository': repo_url, 'author': '{} <{}>'.format(payload['actor']['display_name'], - payload['actor']['username']), + payload['actor']['nickname']), 'comments': 'Bitbucket Cloud commit {}'.format(commit_hash), 'branch': branch, 'project': project, @@ -139,6 +142,10 @@ def handle_pullrequest_rejected(self, payload): def handle_pullrequest(self, payload, refname, category): pr_number = int(payload['pullrequest']['id']) repo_url = payload['repository']['links']['self']['href'] + if payload['repository'].get('project'): + project = payload['repository']['project']['name'] + else: + project = 'none' change = { 'revision': payload['pullrequest']['fromRef']['commit']['hash'], 'revlink': payload['pullrequest']['link'], @@ -147,7 +154,7 @@ def handle_pullrequest(self, payload, refname, category): payload['actor']['username']), 'comments': 'Bitbucket Cloud Pull Request #{}'.format(pr_number), 'branch': refname, - 'project': payload['repository']['project']['name'], + 'project': project, 'category': category, 'properties': {'pullrequesturl': payload['pullrequest']['link']} } From fc7d39be6afae2ba5d44ceadd85bf88b8a9095c4 Mon Sep 17 00:00:00 2001 From: Pawel Krawczyk <616047+kravietz@users.noreply.github.com> Date: Fri, 2 Aug 2019 10:50:11 +0100 Subject: [PATCH 2/8] Add newfragment --- master/buildbot/newsfragments/fix_bitbucketcloud_hook.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 master/buildbot/newsfragments/fix_bitbucketcloud_hook.bugfix diff --git a/master/buildbot/newsfragments/fix_bitbucketcloud_hook.bugfix b/master/buildbot/newsfragments/fix_bitbucketcloud_hook.bugfix new file mode 100644 index 000000000000..f398960580b3 --- /dev/null +++ b/master/buildbot/newsfragments/fix_bitbucketcloud_hook.bugfix @@ -0,0 +1 @@ +:issue:`4873`: Fix Bitbucket Cloud hook crash due to changes in their API. From 3285cfa439dd7968a5d47ddf62856a03f6eae4cb Mon Sep 17 00:00:00 2001 From: Pawel Krawczyk <616047+kravietz@users.noreply.github.com> Date: Fri, 2 Aug 2019 10:53:50 +0100 Subject: [PATCH 3/8] Reflect recent changes in Bitbucket Cloud API --- .../unit/test_www_hooks_bitbucketcloud.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/master/buildbot/test/unit/test_www_hooks_bitbucketcloud.py b/master/buildbot/test/unit/test_www_hooks_bitbucketcloud.py index a19d511b82fd..8483c3e19981 100644 --- a/master/buildbot/test/unit/test_www_hooks_bitbucketcloud.py +++ b/master/buildbot/test/unit/test_www_hooks_bitbucketcloud.py @@ -31,7 +31,7 @@ pushJsonPayload = """ { "actor": { - "username": "John", + "nickname": "John", "display_name": "John Smith" }, "repository": { @@ -52,7 +52,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -87,7 +87,7 @@ pullRequestCreatedJsonPayload = """ { "actor": { - "username": "John", + "nickname": "John", "display_name": "John Smith" }, "pullrequest": { @@ -111,7 +111,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -143,7 +143,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -175,7 +175,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -186,7 +186,7 @@ pullRequestUpdatedJsonPayload = """ { "actor": { - "username": "John", + "nickname": "John", "display_name": "John Smith" }, "pullrequest": { @@ -210,7 +210,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -242,7 +242,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -274,7 +274,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -285,7 +285,7 @@ pullRequestRejectedJsonPayload = """ { "actor": { - "username": "John", + "nickname": "John", "display_name": "John Smith" }, "pullrequest": { @@ -309,7 +309,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -341,7 +341,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -373,7 +373,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -384,7 +384,7 @@ pullRequestFulfilledJsonPayload = """ { "actor": { - "username": "John", + "nickname": "John", "display_name": "John Smith" }, "pullrequest": { @@ -408,7 +408,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -440,7 +440,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -472,7 +472,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -483,7 +483,7 @@ deleteTagJsonPayload = """ { "actor": { - "username": "John", + "nickname": "John", "display_name": "John Smith" }, "repository": { @@ -504,7 +504,7 @@ "ownerName": "BUIL", "public": false, "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -532,7 +532,7 @@ deleteBranchJsonPayload = """ { "actor": { - "username": "John", + "nickname": "John", "display_name": "John Smith" }, "repository": { @@ -553,7 +553,7 @@ "ownerName": "CI", "public": false, "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" @@ -581,7 +581,7 @@ newTagJsonPayload = """ { "actor": { - "username": "John", + "nickname": "John", "display_name": "John Smith" }, "repository": { @@ -602,7 +602,7 @@ "public": false, "ownerName": "CI", "owner": { - "username": "CI", + "nickname": "CI", "display_name": "CI" }, "fullName": "CI/py-repo" From 5bc59ace008e80be2514e309d78f6074545fe665 Mon Sep 17 00:00:00 2001 From: Pawel Krawczyk <616047+kravietz@users.noreply.github.com> Date: Sat, 3 Aug 2019 10:31:54 +0100 Subject: [PATCH 4/8] Simplify project name retrieval --- master/buildbot/www/hooks/bitbucketcloud.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/master/buildbot/www/hooks/bitbucketcloud.py b/master/buildbot/www/hooks/bitbucketcloud.py index 3a13d4201eef..8b55c91669a6 100644 --- a/master/buildbot/www/hooks/bitbucketcloud.py +++ b/master/buildbot/www/hooks/bitbucketcloud.py @@ -70,10 +70,7 @@ def _get_payload(self, request): def handle_repo_push(self, payload): changes = [] - if payload['repository'].get('project'): - project = payload['repository']['project']['name'] - else: - project = 'none' + project = payload['repository'].get('project', {'name': 'none'})['name'] repo_url = payload['repository']['links']['self']['href'] web_url = payload['repository']['links']['html']['href'] @@ -142,10 +139,7 @@ def handle_pullrequest_rejected(self, payload): def handle_pullrequest(self, payload, refname, category): pr_number = int(payload['pullrequest']['id']) repo_url = payload['repository']['links']['self']['href'] - if payload['repository'].get('project'): - project = payload['repository']['project']['name'] - else: - project = 'none' + project = payload['repository'].get('project', {'name': 'none'})['name'] change = { 'revision': payload['pullrequest']['fromRef']['commit']['hash'], 'revlink': payload['pullrequest']['link'], From ca880c56f7f338ee03bb18b58b68c35f25c544ec Mon Sep 17 00:00:00 2001 From: Pawel Krawczyk <616047+kravietz@users.noreply.github.com> Date: Sat, 3 Aug 2019 12:22:25 +0100 Subject: [PATCH 5/8] Remove one more occurence of `username` field --- master/buildbot/www/hooks/bitbucketcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/www/hooks/bitbucketcloud.py b/master/buildbot/www/hooks/bitbucketcloud.py index 8b55c91669a6..f2655fd529db 100644 --- a/master/buildbot/www/hooks/bitbucketcloud.py +++ b/master/buildbot/www/hooks/bitbucketcloud.py @@ -145,7 +145,7 @@ def handle_pullrequest(self, payload, refname, category): 'revlink': payload['pullrequest']['link'], 'repository': repo_url, 'author': '{} <{}>'.format(payload['actor']['display_name'], - payload['actor']['username']), + payload['actor']['nickname']), 'comments': 'Bitbucket Cloud Pull Request #{}'.format(pr_number), 'branch': refname, 'project': project, From 3b4a0ccee98f98bab3d4c178d17ddfe68907572d Mon Sep 17 00:00:00 2001 From: Rajdeep Bharati Date: Mon, 5 Aug 2019 16:30:15 +0530 Subject: [PATCH 6/8] Create new changebuilds page Displays all builds for the selected change. --- www/base/src/app/app.module.js | 2 + .../changebuilds/changebuilds.controller.js | 40 +++++++++++++++++++ .../changebuilds/changebuilds.route.js | 35 ++++++++++++++++ .../changebuilds/changebuilds.tpl.jade | 5 +++ .../src/app/changes/changes.controller.js | 5 ++- www/base/src/app/changes/changes.route.js | 8 ++-- .../directives/changelist/changelist.tpl.jade | 2 + 7 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 www/base/src/app/changes/changebuilds/changebuilds.controller.js create mode 100644 www/base/src/app/changes/changebuilds/changebuilds.route.js create mode 100644 www/base/src/app/changes/changebuilds/changebuilds.tpl.jade diff --git a/www/base/src/app/app.module.js b/www/base/src/app/app.module.js index 5492b1cf72a5..7532be4f6b1e 100644 --- a/www/base/src/app/app.module.js +++ b/www/base/src/app/app.module.js @@ -50,6 +50,8 @@ require('./builders/step/step.controller.js'); require('./builders/step/step.route.js'); require('./buildrequests/pendingbuildrequests.controller.js'); require('./buildrequests/pendingbuildrequests.route.js'); +require('./changes/changebuilds/changebuilds.controller.js'); +require('./changes/changebuilds/changebuilds.route.js'); require('./changes/changes.controller.js'); require('./changes/changes.route.js'); require('./common/common.constant.js'); diff --git a/www/base/src/app/changes/changebuilds/changebuilds.controller.js b/www/base/src/app/changes/changebuilds/changebuilds.controller.js new file mode 100644 index 000000000000..5c3072822a8f --- /dev/null +++ b/www/base/src/app/changes/changebuilds/changebuilds.controller.js @@ -0,0 +1,40 @@ +class ChangeBuildsController { + constructor($scope, dataService, bbSettingsService, $stateParams, resultsService, $interval, restService) { + + _.mixin($scope, resultsService); + $scope.settings = bbSettingsService.getSettingsGroup('ChangeBuilds'); + $scope.$watch('settings', () => bbSettingsService.save() + , true); + const buildsFetchLimit = $scope.settings.buildsFetchLimit.value; + + const dataAccessor = dataService.open().closeOnDestroy($scope); + $scope.builders = dataAccessor.getBuilders(); + + const changeId = $scope.changeId = $stateParams.changeid; + + dataAccessor.getChanges(changeId).onNew = function(change) { + $scope.change = change; + } + + const getBuildsData = function() { + let requestUrl = `changes/${changeId}/builds`; + if (!buildsFetchLimit == '') { + requestUrl = `changes/${changeId}/builds?limit=${buildsFetchLimit}`; + } + restService.get(requestUrl).then((data) => { + $scope.builds = data.builds; + }); + } + + getBuildsData(); + + const stop = $interval(() => { + getBuildsData(); + }, 5000); + + $scope.$on('$destroy', () => $interval.cancel(stop)); + } +} + +angular.module('app') +.controller('changebuildsController', ['$scope', 'dataService', 'bbSettingsService', '$stateParams', 'resultsService', '$interval', 'restService', ChangeBuildsController]); diff --git a/www/base/src/app/changes/changebuilds/changebuilds.route.js b/www/base/src/app/changes/changebuilds/changebuilds.route.js new file mode 100644 index 000000000000..532e780bae27 --- /dev/null +++ b/www/base/src/app/changes/changebuilds/changebuilds.route.js @@ -0,0 +1,35 @@ +class ChangeBuildsState { + constructor($stateProvider, bbSettingsServiceProvider) { + // Name of the state + const name = 'changebuilds'; + + // Configuration + const cfg = {} + + // Register new state + const state = { + controller: `${name}Controller`, + template: require('./changebuilds.tpl.jade'), + name, + url: '/changes/:changeid', + data: cfg, + reloadOnSearch: false + } + + $stateProvider.state(state); + + bbSettingsServiceProvider.addSettingsGroup({ + name:'ChangeBuilds', + caption: 'ChangeBuilds page related settings', + items:[{ + type:'integer', + name:'buildsFetchLimit', + caption:'Maximum number of builds to fetch for the selected change', + default_value: '' + }] + }); + } +} + +angular.module('app') +.config(['$stateProvider', 'bbSettingsServiceProvider', ChangeBuildsState]); diff --git a/www/base/src/app/changes/changebuilds/changebuilds.tpl.jade b/www/base/src/app/changes/changebuilds/changebuilds.tpl.jade new file mode 100644 index 000000000000..5596c2f14c56 --- /dev/null +++ b/www/base/src/app/changes/changebuilds/changebuilds.tpl.jade @@ -0,0 +1,5 @@ +.container + .row + changedetails(change="change") + div(ng-if="builds") + builds-table(builds="builds", builders="builders", ng-if="builds") diff --git a/www/base/src/app/changes/changes.controller.js b/www/base/src/app/changes/changes.controller.js index 079f614893a7..10c4d54d9c9d 100644 --- a/www/base/src/app/changes/changes.controller.js +++ b/www/base/src/app/changes/changes.controller.js @@ -1,5 +1,6 @@ class Changes { - constructor($log, $scope, dataService, bbSettingsService) { + constructor($log, $scope, dataService, bbSettingsService, + $location, $rootScope) { $scope.settings = bbSettingsService.getSettingsGroup("Changes"); $scope.$watch('settings', () => bbSettingsService.save() , true); @@ -13,4 +14,4 @@ class Changes { angular.module('app') -.controller('changesController', ['$log', '$scope', 'dataService', 'bbSettingsService', Changes]); +.controller('changesController', ['$log', '$scope', 'dataService', 'bbSettingsService', '$location', '$rootScope', Changes]); diff --git a/www/base/src/app/changes/changes.route.js b/www/base/src/app/changes/changes.route.js index 42ccc911f72c..f61a7783e899 100644 --- a/www/base/src/app/changes/changes.route.js +++ b/www/base/src/app/changes/changes.route.js @@ -15,8 +15,9 @@ class ChangesState { controller: `${name}Controller`, template: require('./changes.tpl.jade'), name, - url: '/changes', - data: cfg + url: '/changes?id', + data: cfg, + reloadOnSearch: false }; $stateProvider.state(state); @@ -29,8 +30,7 @@ class ChangesState { name:'changesFetchLimit', caption:'Maximum number of changes to fetch', default_value: 50 - } - ]}); + }]}); } } diff --git a/www/base/src/app/common/directives/changelist/changelist.tpl.jade b/www/base/src/app/common/directives/changelist/changelist.tpl.jade index f20a1b58dd1f..bfb813e4ccc2 100644 --- a/www/base/src/app/common/directives/changelist/changelist.tpl.jade +++ b/www/base/src/app/common/directives/changelist/changelist.tpl.jade @@ -15,4 +15,6 @@ .row ul.list-group li.list-group-item(ng-repeat="change in changes") + a(ui-sref="changebuilds({changeid: change.changeid})") + | See builds changedetails(change="change") From e7ba33499f7338d695e12df251c0130a9923e22e Mon Sep 17 00:00:00 2001 From: PhilippSelenium <31542906+PhilippSelenium@users.noreply.github.com> Date: Mon, 5 Aug 2019 15:12:34 +0200 Subject: [PATCH 7/8] Buildbot worker dockerfile from ubuntu 18.04 1. dumb-init can be installed from apt 2. pip can be installed from apt --- worker/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/worker/Dockerfile b/worker/Dockerfile index 244fb1d3c427..1b6ded2330e8 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -6,7 +6,7 @@ # Provides a base Ubuntu (16.04) image with latest buildbot worker installed # the worker image is not optimized for size, but rather uses ubuntu for wider package availability -FROM ubuntu:16.04 +FROM ubuntu:18.04 MAINTAINER Buildbot maintainers @@ -28,14 +28,13 @@ RUN apt-get update && \ libffi-dev \ libssl-dev \ python-setuptools \ - curl && \ - rm -rf /var/lib/apt/lists/* && \ + python-pip \ # Test runs produce a great quantity of dead grandchild processes. In a # non-docker environment, these are automatically reaped by init (process 1), # so we need to simulate that here. See https://github.com/Yelp/dumb-init - curl https://github.com/Yelp/dumb-init/releases/download/v1.2.1/dumb-init_1.2.1_amd64.deb -Lo /tmp/init.deb && dpkg -i /tmp/init.deb &&\ - # ubuntu pip version has issues so we should use the official upstream version it: https://github.com/pypa/pip/pull/3287 - easy_install pip && \ + dumb-init \ + curl && \ + rm -rf /var/lib/apt/lists/* && \ # Install required python packages, and twisted pip --no-cache-dir install 'twisted[tls]' && \ pip install virtualenv && \ From e60b11432495b3610ab2733c9b21725577fbd465 Mon Sep 17 00:00:00 2001 From: PhilippSelenium <31542906+PhilippSelenium@users.noreply.github.com> Date: Mon, 5 Aug 2019 15:20:32 +0200 Subject: [PATCH 8/8] Added newsfragment --- .../buildbot/newsfragments/buildbot-worker-ubuntu-18.04.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 master/buildbot/newsfragments/buildbot-worker-ubuntu-18.04.feature diff --git a/master/buildbot/newsfragments/buildbot-worker-ubuntu-18.04.feature b/master/buildbot/newsfragments/buildbot-worker-ubuntu-18.04.feature new file mode 100644 index 000000000000..5712431a2175 --- /dev/null +++ b/master/buildbot/newsfragments/buildbot-worker-ubuntu-18.04.feature @@ -0,0 +1 @@ +:issue:`4928`: Update buildbot worker image to ubuntu 18.04