Merge pull request #2348 from tardyp/porttostable

rc2 preparations

common/maketarballs.sh

@@ -9,7 +9,8 @@ do
     cd ${pkg}
     rm -rf MANIFEST dist
     python setup.py sdist
+    # wheels must be build separatly in order to properly omit tests
+    python setup.py bdist_wheel
   )
   cp ${pkg}/dist/* dist/
-  pip wheel ${pkg} -w dist
 done

common/porttostable.py

@@ -0,0 +1,34 @@
+import os
+from subprocess import CalledProcessError
+from subprocess import check_output
+
+import requests
+import yaml
+
+s = requests.Session()
+with open(os.path.expanduser('~/.config/hub')) as f:
+    config = yaml.load(f)['github.com'][0]
+    s.auth = config['user'], config['oauth_token']
+
+os.system("git fetch --all")
+r = s.get("https://api.github.com/search/issues?q=label:\"port%20to%20stable\"+repo:buildbot/buildbot")
+to_port = r.json()
+summary = ""
+for pr in to_port['items']:
+    r = s.get("https://api.github.com/repos/buildbot/buildbot/pulls/{number}/commits".format(**pr))
+    commits = r.json()
+    for c in commits:
+        title = c['commit']['message'].split("\n")[0]
+        try:
+            check_output("git cherry-pick {sha} 2>&1".format(**c), shell=True)
+        except CalledProcessError as e:
+            os.system("git diff")
+            os.system("git reset --hard HEAD 2>&1 >/dev/null")
+            if '--allow-empty' in e.output:
+                continue
+            if 'fatal: bad object' in e.output:
+                continue
+            print "cannot automatically cherry-pick", pr['number'], c['sha'], title,e.output
+        else:
+            summary += "\n#{number}: {title}".format(number=pr['number'], title=title , **c)
+print summary

common/smokedist.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+
+set -e
+for suffix in whl tar.gz
+do
+    VE=sandbox.$suffix
+    rm -rf $VE
+    virtualenv $VE
+    . $VE/bin/activate
+    pip install -U pip
+    pip install  mock
+    pip install dist/buildbot-0*.$suffix
+    pip install dist/buildbot?pkg*.$suffix
+    pip install dist/*.$suffix
+    smokes/run.sh
+done

master/buildbot/changes/bitbucket.py

@@ -102,7 +102,8 @@ def _processChanges(self, page):
             if not self.branch or branch in self.branch:
                 current = yield self._getCurrentRev(nr)
 
-                if not current or current != revision:
+                # compare _short_ hashes to check if the PR has been updated
+                if not current or current[0:12] != revision[0:12]:
                     # parse pull request api page (required for the filter)
                     page = yield client.getPage(str(pr['links']['self']['href']))
                     pr_json = json.loads(page, encoding=self.encoding)

master/buildbot/reporters/message.py

@@ -134,4 +134,4 @@ def __call__(self, mode, buildername, buildset, build, master, previous_results,
                    sourcestamps=self.messageSourceStamps(ss_list)
                    )
         contents = tpl.render(cxt)
-        return {'body': contents, 'type': 'plain'}
+        return {'body': contents, 'type': self.template_type}

master/buildbot/spec/types/logchunk.raml

@@ -23,6 +23,18 @@ description: |
     These are specified as query parameters via the REST interface, or as arguments to the :py:meth:`~buildbot.data.connector.DataConnector.get` method in Python.
     The result will begin with line ``offset`` (so the resulting ``firstline`` will be equal to the given ``offset``), and will contain up to ``limit`` lines.
 
+    Following example will get the first 100 lines of a log::
+
+        from buildbot.data import resultspec
+        first_100_lines = yield self.master.data.get(("logs", log['logid'], "contents"),
+            resultSpec=resultspec.ResultSpec(limit=100))
+
+    Following example will get the last 100 lines of a log::
+
+        from buildbot.data import resultspec
+        last_100_lines = yield self.master.data.get(("logs", log['logid'], "contents"),
+            resultSpec=resultspec.ResultSpec(offset=log['num_lines']-100))
+
     .. note::
 
         There is no event for a new chunk. Instead, the log resource is updated when new chunks are added, with the new number of lines.

master/buildbot/test/unit/test_changes_bitbucket.py

@@ -26,13 +26,14 @@
 
 
 class SourceRest():
+    """https://bitbucket.org/!api/2.0/repositories/{owner}/{slug}"""
     template = """\
 {
 
     "hash": "%(hash)s",
     "links": {
         "html": {
-            "href": "https://bitbucket.org/%(owner)s/%(slug)s/commits/%(hash)s"
+            "href": "https://bitbucket.org/%(owner)s/%(slug)s/commits/%(short_hash)s"
         }
     },
     "repository": {
@@ -67,6 +68,7 @@ def request(self):
             "owner": self.owner,
             "slug": self.slug,
             "hash": self.hash,
+            "short_hash": self.hash[0:12],
             "date": self.date,
         }
 
@@ -78,6 +80,7 @@ def repo_request(self):
 
 
 class PullRequestRest():
+    """https://bitbucket.org/!api/2.0/repositories/{owner}/{slug}/pullrequests/{pull_request_id}"""
     template = """\
 {
 
@@ -124,6 +127,7 @@ def request(self):
             "description": self.description,
             "title": self.title,
             "hash": self.source.hash,
+            "short_hash": self.source.hash[0:12],
             "owner": self.source.owner,
             "slug": self.source.slug,
             "display_name": self.display_name,
@@ -134,6 +138,7 @@ def request(self):
 
 
 class PullRequestListRest():
+    """https://bitbucket.org/api/2.0/repositories/{owner}/{slug}/pullrequests"""
     template = """\
         {
             "description": "%(description)s",
@@ -151,10 +156,10 @@ class PullRequestListRest():
             "title": "%(title)s",
             "source": {
                 "commit": {
-                    "hash": "%(hash)s",
+                    "hash": "%(short_hash)s",
                     "links": {
                         "self": {
-                            "href": "https://bitbucket.org/!api/2.0/repositories/%(src_owner)s/%(src_slug)s/commit/%(hash)s"
+                            "href": "https://bitbucket.org/!api/2.0/repositories/%(src_owner)s/%(src_slug)s/commit/%(short_hash)s"
                         }
                     }
                 },
@@ -200,6 +205,7 @@ def request(self):
                 "display_name": pr.display_name,
                 "title": pr.title,
                 "hash": pr.source.hash,
+                "short_hash": pr.source.hash[0:12],
                 "src_owner": pr.source.owner,
                 "src_slug": pr.source.slug,
                 "created_on": pr.created_on,
@@ -260,7 +266,7 @@ def setUp(self):
         src = SourceRest(
             owner="contributor",
             slug="slug",
-            hash="000000000000000000000000000001",
+            hash="1111111111111111111111111111111111111111",
             date=self.date,
         )
         pr = PullRequestRest(
@@ -280,7 +286,7 @@ def setUp(self):
         src = SourceRest(
             owner="contributor",
             slug="slug",
-            hash="000000000000000000000000000002",
+            hash="2222222222222222222222222222222222222222",
             date=self.date,
         )
         pr = PullRequestRest(
@@ -367,8 +373,8 @@ def test_poll_new_pull_requests(self):
             'project': u'',
             'properties': {},
             'repository': u'https://bitbucket.org/contributor/slug',
-            'revision': u'000000000000000000000000000001',
-            'revlink': u'https://bitbucket.org/contributor/slug/commits/000000000000000000000000000001',
+            'revision': u'1111111111111111111111111111111111111111',
+            'revlink': u'https://bitbucket.org/contributor/slug/commits/111111111111',
             'src': u'bitbucket',
             'when_timestamp': 1381869500,
         }])
@@ -392,8 +398,8 @@ def test_poll_no_updated_pull_request(self):
             'project': u'',
             'properties': {},
             'repository': u'https://bitbucket.org/contributor/slug',
-            'revision': u'000000000000000000000000000001',
-            'revlink': u'https://bitbucket.org/contributor/slug/commits/000000000000000000000000000001',
+            'revision': u'1111111111111111111111111111111111111111',
+            'revlink': u'https://bitbucket.org/contributor/slug/commits/111111111111',
             'src': u'bitbucket',
             'when_timestamp': 1381869500,
         }])
@@ -420,8 +426,9 @@ def test_poll_updated_pull_request(self):
             'project': u'',
             'properties': {},
             'repository': u'https://bitbucket.org/contributor/slug',
-            'revision': u'000000000000000000000000000001',
-            'revlink': u'https://bitbucket.org/contributor/slug/commits/000000000000000000000000000001',
+
+            'revision': u'1111111111111111111111111111111111111111',
+            'revlink': u'https://bitbucket.org/contributor/slug/commits/111111111111',
             'src': u'bitbucket',
             'when_timestamp': 1381869500,
         }])
@@ -439,8 +446,8 @@ def test_poll_updated_pull_request(self):
                 'project': u'',
                 'properties': {},
                 'repository': u'https://bitbucket.org/contributor/slug',
-                'revision': u'000000000000000000000000000001',
-                'revlink': u'https://bitbucket.org/contributor/slug/commits/000000000000000000000000000001',
+                'revision': u'1111111111111111111111111111111111111111',
+                'revlink': u'https://bitbucket.org/contributor/slug/commits/111111111111',
                 'src': u'bitbucket',
                 'when_timestamp': 1381869500,
             },
@@ -454,8 +461,8 @@ def test_poll_updated_pull_request(self):
                 'project': u'',
                 'properties': {},
                 'repository': u'https://bitbucket.org/contributor/slug',
-                'revision': u'000000000000000000000000000002',
-                'revlink': u'https://bitbucket.org/contributor/slug/commits/000000000000000000000000000002',
+                'revision': u'2222222222222222222222222222222222222222',
+                'revlink': u'https://bitbucket.org/contributor/slug/commits/222222222222',
                 'src': u'bitbucket',
                 'when_timestamp': 1381869500,
             }
@@ -499,8 +506,8 @@ def test_poll_pull_request_filter_True(self):
             'project': u'',
             'properties': {},
             'repository': u'https://bitbucket.org/contributor/slug',
-            'revision': u'000000000000000000000000000001',
-            'revlink': u'https://bitbucket.org/contributor/slug/commits/000000000000000000000000000001',
+            'revision': u'1111111111111111111111111111111111111111',
+            'revlink': u'https://bitbucket.org/contributor/slug/commits/111111111111',
             'src': u'bitbucket',
             'when_timestamp': 1381869500,
         }])
@@ -528,8 +535,8 @@ def test_poll_pull_request_not_useTimestamps(self):
             'project': u'',
             'properties': {},
             'repository': u'https://bitbucket.org/contributor/slug',
-            'revision': u'000000000000000000000000000001',
-            'revlink': u'https://bitbucket.org/contributor/slug/commits/000000000000000000000000000001',
+            'revision': u'1111111111111111111111111111111111111111',
+            'revlink': u'https://bitbucket.org/contributor/slug/commits/111111111111',
             'src': u'bitbucket',
             'when_timestamp': 1396825656,
         }])

master/buildbot/test/unit/test_worker_docker.py

@@ -86,14 +86,14 @@ def test_start_instance_volume_renderable(self):
         bs = self.ConcreteWorker('bot', 'pass', 'tcp://1234:2375', 'worker', ['bin/bash'],
                                  volumes=[Interpolate('/data:/buildslave/%(kw:builder)s/build', builder=Property('builder'))])
         id, name = yield bs.start_instance(self.build)
-        self.assertEqual(bs.volumes, ['/data:/buildslave/docker_worker/build'])
+        self.assertEqual(bs.volumes, ['/buildslave/docker_worker/build'])
 
     @defer.inlineCallbacks
     def test_volume_no_suffix(self):
         bs = self.ConcreteWorker(
             'bot', 'pass', 'tcp://1234:2375', 'worker', ['bin/bash'], volumes=['/src/webapp:/opt/webapp'])
         yield bs.start_instance(self.build)
-        self.assertEqual(bs.volumes, ['/src/webapp:/opt/webapp'])
+        self.assertEqual(bs.volumes, ['/opt/webapp'])
         self.assertEqual(
             bs.binds, {'/src/webapp': {'bind': '/opt/webapp', 'ro': False}})
 
@@ -104,7 +104,7 @@ def test_volume_ro_rw(self):
                                           '~:/backup:rw'])
         yield bs.start_instance(self.build)
         self.assertEqual(
-            bs.volumes, ['/src/webapp:/opt/webapp:ro', '~:/backup:rw'])
+            bs.volumes, ['/opt/webapp', '/backup'])
         self.assertEqual(bs.binds, {'/src/webapp': {'bind': '/opt/webapp', 'ro': True},
                                     '~': {'bind': '/backup', 'ro': False}})
 

master/buildbot/worker/docker.py

@@ -80,7 +80,7 @@ def __init__(self, name, password, docker_host, image=None, command=None,
                 if not isinstance(volume_string, str):
                     continue
                 try:
-                    volume, bind = volume_string.split(":", 1)
+                    bind, volume = volume_string.split(":", 1)
                 except ValueError:
                     config.error("Invalid volume definition for docker "
                                  "%s. Skipping..." % volume_string)
@@ -110,18 +110,19 @@ def parse_volumes(self, volumes):
         self.volumes = []
         for volume_string in (volumes or []):
             try:
-                volume, bind = volume_string.split(":", 1)
+                bind, volume = volume_string.split(":", 1)
             except ValueError:
                 config.error("Invalid volume definition for docker "
                              "%s. Skipping..." % volume_string)
                 continue
-            self.volumes.append(volume_string)
 
             ro = False
-            if bind.endswith(':ro') or bind.endswith(':rw'):
-                ro = bind[-2:] == 'ro'
-                bind = bind[:-3]
-            self.binds[volume] = {'bind': bind, 'ro': ro}
+            if volume.endswith(':ro') or volume.endswith(':rw'):
+                ro = volume[-2:] == 'ro'
+                volume = volume[:-3]
+
+            self.volumes.append(volume)
+            self.binds[bind] = {'bind': volume, 'ro': ro}
 
     def createEnvironment(self):
         result = {

master/docs/relnotes/0.9.0rc2.rst

@@ -0,0 +1,72 @@
+Release Notes for Buildbot ``0.9.0rc2``
+========================================
+
+
+The following are the release notes for Buildbot ``0.9.0rc2``.
+This version was released on August 23, 2016.
+
+See :ref:`Upgrading to Nine` for a guide to upgrading from 0.8.x to 0.9.x
+
+
+Master
+------
+
+Features
+~~~~~~~~
+
+* add a UI button to allow to cancel the whole queue for a builder
+
+Fixes
+~~~~~
+
+* fix the UI to allow to cancel a buildrequest (:bug:`3582`)
+* Fix BitbucketPullrequestPoller change detection
+* Fix customization for template_type in email reporter
+* fix DockerLatent integration of volumes mounting
+* misc doc fixes
+* fix buildbot not booting when builder tags contains duplicates
+* ``forcesched``: fix owner parameter when no authentication is used
+* REST: fix problem with twisted 16 error reporting
+* CORS: format errors according to API type
+* Dockerfiles fix and upgrade Ubuntu to 16.04
+* Fixes #3430 Increased size of builder identifier from 20 to 50 (brings it in line to size of steps and workers in same module).
+* Fix missing VS2015 entry_points
+* removed the restriction on twisted < 16.3.0 now that autobahn 0.16.0 fixed the issue
+
+Changes for Developers
+~~~~~~~~~~~~~~~~~~~~~~
+
+Features
+~~~~~~~~
+
+Fixes
+~~~~~
+
+
+Deprecations, Removals, and Non-Compatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* remove repo from worker code (obsoleted by repo master source step)
+
+
+Worker
+------
+
+Fixes
+~~~~~
+
+Changes for Developers
+~~~~~~~~~~~~~~~~~~~~~~
+
+Deprecations, Removals, and Non-Compatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+Details
+-------
+
+For a more detailed description of the changes made in this version, see the git log itself:
+
+.. code-block:: bash
+
+   git log v0.9.0rc1..v0.9.0rc2