Merge pull request #2811 from rodrigc/rest3

Fix test_www_rest on Python 3

.py3.notworking.txt

@@ -1,6 +1,5 @@
 buildbot.test.integration.test_URLs.UrlForBuildMaster.test_url
 buildbot.test.integration.test_trigger.TriggeringMaster.test_trigger
-buildbot.test.integration.test_www.Www.test_masters
 buildbot.test.unit.test_data_properties.Properties.test_setBuildProperties
 buildbot.test.unit.test_reporters_words.TestContactChannel.test_command_watch_builder0_get_notifications
 buildbot.test.unit.test_schedulers_dependent.Dependent.test_related_buildset_FAILURE
@@ -9,44 +8,4 @@ buildbot.test.unit.test_schedulers_dependent.Dependent.test_related_buildset_WAR
 buildbot.test.unit.test_schedulers_dependent.Dependent.test_unrelated_buildset
 buildbot.test.unit.test_schedulers_triggerable.Triggerable.test_trigger
 buildbot.test.unit.test_schedulers_triggerable.Triggerable.test_trigger_overlapping
-buildbot.test.unit.test_schedulers_trysched.Try_Jobdir.test_parseJob_v5
-buildbot.test.unit.test_schedulers_trysched.Try_Jobdir.test_parseJob_v5_invalid_json
-buildbot.test.unit.test_schedulers_trysched.Try_Jobdir.test_parseJob_v5_no_builders
-buildbot.test.unit.test_schedulers_trysched.Try_Jobdir.test_parseJob_v5_no_properties
 buildbot.test.unit.test_www_config.IndexResource.test_render
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_fields
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_filter_and_order
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_filter_and_order_desc
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_filter_pagination
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_limit
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_limit_at_end
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_limit_past_end
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_list_filter
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_offset
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_offset_limit
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_offset_past_end
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_operator_filter
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_order
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_order_desc
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_simple_filter
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_details_none
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_decode_result_spec_not_a_collection_properties
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_decode_result_spec_properties
-buildbot.test.unit.test_www_rest.V2RootResource_REST.test_raw
-buildbot.test.unit.test_www_service.Test.test_reconfigService_expiration_time
-buildbot.test.unit.test_www_service.Test.test_reconfigService_port
-buildbot.test.unit.test_www_service.Test.test_reconfigService_port_changes
-buildbot.test.unit.test_www_service.Test.test_reconfigService_port_changes_to_none
-buildbot.test.unit.test_www_service.Test.test_reconfigService_reconfigResources
-buildbot.test.unit.test_www_service.Test.test_setupSiteWithHook
-buildbot.test.unit.test_www_service.Test.test_setupSiteWithHookAndAuth
-buildbot.test.unit.test_www_service.TestBuildbotSite.test_getSession_from_bad_jwt
-buildbot.test.unit.test_www_service.TestBuildbotSite.test_getSession_from_correct_jwt
-buildbot.test.unit.test_www_service.TestBuildbotSite.test_getSession_from_expired_jwt
-buildbot.test.unit.test_www_service.TestBuildbotSite.test_getSession_with_no_user_info
-buildbot.test.unit.test_www_service.TestBuildbotSite.test_updateSession
-buildbot.test.unit.test_www_sse.EventResource.test_listen
-buildbot.test.unit.test_www_sse.EventResource.test_listen_add_then_close
-buildbot.test.unit.test_www_sse.EventResource.test_listen_add_then_remove
-buildbot.test.unit.test_www_sse.EventResource.test_simpleapi

master/buildbot/data/resultspec.py

@@ -173,9 +173,10 @@ def __init__(self, filters=None, fields=None, properties=None, order=None,
         self.fieldMapping = {}
 
     def __repr__(self):
-        return "ResultSpec(**" + repr(dict(filters=self.filters, fields=self.fields,
-                                           properties=self.properties,
-                                           order=self.order, limit=self.limit, offset=self.offset)) + ")"
+        return ("ResultSpec(**{{'filters': {}, 'fields': {}, 'properties': {}, "
+                "'order': {}, 'limit': {}, 'offset': {}").format(
+                self.filters, self.fields, self.properties, self.order,
+                self.limit, self.offset) + "})"
 
     def __eq__(self, b):
         for i in ['filters', 'fields', 'properties', 'order', 'limit', 'offset']:

master/buildbot/db/migrate/versions/049_add_schedulers_enabled.py

@@ -17,12 +17,14 @@
 from __future__ import print_function
 
 import sqlalchemy as sa
+
 from buildbot.util import sautils
 
 
 def upgrade(migrate_engine):
     metadata = sa.MetaData()
     metadata.bind = migrate_engine
     schedulers_table = sautils.Table('schedulers', metadata, autoload=True)
-    enabled = sa.Column('enabled', sa.SmallInteger, nullable=False, server_default="1")
+    enabled = sa.Column('enabled', sa.SmallInteger,
+                        nullable=False, server_default="1")
     enabled.create(schedulers_table)

master/buildbot/db/pool.py

@@ -32,7 +32,6 @@
 from buildbot.db.schedulers import SchedulerAlreadyClaimedError
 from buildbot.process import metrics
 
-
 # set this to True for *very* verbose query debugging output; this can
 # be monkey-patched from master.cfg, too:
 #     from buildbot.db import pool
@@ -146,7 +145,8 @@ def _start(self):
 
     def _stop(self):
         self._stop_evt = None
-        threads.deferToThreadPool(self.reactor, self._pool, self.engine.dispose)
+        threads.deferToThreadPool(
+            self.reactor, self._pool, self.engine.dispose)
         self._pool.stop()
         self.running = False
 
@@ -201,7 +201,8 @@ def __thd(self, with_engine, callable, args, kwargs):
                     # and re-try
                     log.err(e, 'retrying {} after sql error {}'.format(callable, e))
                     continue
-                # AlreadyClaimedError are normal especially in a multimaster configuration
+                # AlreadyClaimedError are normal especially in a multimaster
+                # configuration
                 except (AlreadyClaimedError, ChangeSourceAlreadyClaimedError, SchedulerAlreadyClaimedError):
                     raise
                 except Exception as e:

master/buildbot/schedulers/trysched.py

@@ -195,7 +195,8 @@ def postprocess_parsed_job():
             postprocess_parsed_job()
         elif ver == "5":
             try:
-                parsed_job = json.loads(p.strings[0])
+                data = bytes2NativeString(p.strings[0])
+                parsed_job = json.loads(data)
             except ValueError:
                 raise BadJobfile("unable to parse JSON")
             postprocess_parsed_job()

master/buildbot/test/fake/fakedb.py

@@ -38,6 +38,7 @@
 from buildbot.db import changesources
 from buildbot.db import schedulers
 from buildbot.test.util import validation
+from buildbot.util import bytes2NativeString
 from buildbot.util import datetime2epoch
 from buildbot.util import service
 from buildbot.util import unicode2bytes
@@ -1655,7 +1656,7 @@ def setState(self, objectid, name, value):
 
     def atomicCreateState(self, objectid, name, thd_create_callback):
         value = thd_create_callback()
-        self.states[objectid][name] = json.dumps(value)
+        self.states[objectid][name] = json.dumps(bytes2NativeString(value))
         return defer.succeed(value)
 
     # fake methods

master/buildbot/test/fake/fakemq.py

@@ -15,6 +15,7 @@
 
 from __future__ import absolute_import
 from __future__ import print_function
+from future.utils import string_types
 
 from twisted.internet import defer
 
@@ -52,7 +53,10 @@ def produce(self, routingKey, data):
 # routing key
 #        if self.verifyMessages:
 #            validation.verifyMessage(self.testcase, routingKey, data)
-        if any(not isinstance(k, str) for k in routingKey):
+
+        # TODO: the key to a mq should be changed so that it is unicode
+        # (text_type).
+        if any(not isinstance(k, (bytes, string_types)) for k in routingKey):
             raise AssertionError("%s is not all strings" % (routingKey,))
         self.productions.append((routingKey, data))
         # note - no consumers are called: IT'S A FAKE
@@ -69,7 +73,8 @@ def callConsumer(self, routingKey, msg):
             raise AssertionError("no consumer found")
 
     def startConsuming(self, callback, filter, persistent_name=None):
-        if any(not isinstance(k, str) and k is not None for k in filter):
+        if any(not isinstance(k, (bytes, string_types)) and
+               k is not None for k in filter):
             raise AssertionError("%s is not a filter" % (filter,))
         qref = FakeQueueRef()
         qref.qrefs = self.qrefs

master/buildbot/test/integration/test_www.py

@@ -33,6 +33,8 @@
 from buildbot.test.fake import fakemaster
 from buildbot.test.util import db
 from buildbot.test.util import www
+from buildbot.util import bytes2NativeString
+from buildbot.util import unicode2bytes
 from buildbot.www import service as wwwservice
 from buildbot.www import auth
 from buildbot.www import authz
@@ -54,7 +56,7 @@ def dataReceived(self, bytes):
 
     def connectionLost(self, reason):
         if reason.check(client.ResponseDone):
-            self.finishedDeferred.callback(''.join(self.body))
+            self.finishedDeferred.callback(b''.join(self.body))
         else:
             self.finishedDeferred.errback(reason)
 
@@ -103,6 +105,7 @@ def setUp(self):
         # the config.  The second reconfig isn't really required, but doesn't
         # hurt.
         self.url = 'http://127.0.0.1:%d/' % master.www.getPortnum()
+        self.url = unicode2bytes(self.url)
         master.config.buildbotURL = self.url
         yield master.www.reconfigServiceWithBuildbotConfig(master.config)
 
@@ -126,7 +129,7 @@ def tearDown(self):
 
     @defer.inlineCallbacks
     def apiGet(self, url, expect200=True):
-        pg = yield self.agent.request('GET', url)
+        pg = yield self.agent.request(b'GET', url)
 
         # this is kind of obscene, but protocols are like that
         d = defer.Deferred()
@@ -139,10 +142,10 @@ def apiGet(self, url, expect200=True):
         if expect200 and pg.code != 200:
             self.fail("did not get 200 response for '%s'" % (url,))
 
-        defer.returnValue(json.loads(body))
+        defer.returnValue(json.loads(bytes2NativeString(body)))
 
     def link(self, suffix):
-        return self.url + 'api/v2/' + suffix
+        return self.url + b'api/v2/' + suffix
 
     # tests
 
@@ -159,7 +162,7 @@ def test_masters(self):
                           active=1, last_active=OTHERTIME),
         ])
 
-        res = yield self.apiGet(self.link('masters'))
+        res = yield self.apiGet(self.link(b'masters'))
         self.assertEqual(res, {
             'masters': [
                 {'active': False, 'masterid': 7, 'name': 'some:master',
@@ -171,7 +174,7 @@ def test_masters(self):
                 'total': 2,
             }})
 
-        res = yield self.apiGet(self.link('masters/7'))
+        res = yield self.apiGet(self.link(b'masters/7'))
         self.assertEqual(res, {
             'masters': [
                 {'active': False, 'masterid': 7, 'name': 'some:master',

master/buildbot/test/unit/test_data_types.py

@@ -108,24 +108,24 @@ class Boolean(TypeMixin, unittest.TestCase):
     good = [True, False]
     bad = [None, 0, 1]
     stringValues = [
-        ('on', True),
-        ('true', True),
-        ('yes', True),
-        ('1', True),
-        ('off', False),
-        ('false', False),
-        ('no', False),
-        ('0', False),
-        ('ON', True),
-        ('TRUE', True),
-        ('YES', True),
-        ('OFF', False),
-        ('FALSE', False),
-        ('NO', False),
+        (b'on', True),
+        (b'true', True),
+        (b'yes', True),
+        (b'1', True),
+        (b'off', False),
+        (b'false', False),
+        (b'no', False),
+        (b'0', False),
+        (b'ON', True),
+        (b'TRUE', True),
+        (b'YES', True),
+        (b'OFF', False),
+        (b'FALSE', False),
+        (b'NO', False),
     ]
     cmpResults = [
-        (False, 'no', 0),
-        (True, 'true', 0),
+        (False, b'no', 0),
+        (True, b'true', 0),
     ]
 
 

master/buildbot/test/unit/test_db_migrate_versions_049_add_schedulers_enabled.py

@@ -17,6 +17,7 @@
 from __future__ import print_function
 
 import sqlalchemy as sa
+
 from twisted.trial import unittest
 
 from buildbot.test.util import migration
@@ -42,7 +43,8 @@ def create_tables_thd(self, conn):
             # name for this scheduler, as given in the configuration, plus a hash
             # of that name used for a unique index
             sa.Column('name', sa.Text, nullable=False),
-            sa.Column('name_hash', sa.String(40), nullable=False, server_default='')
+            sa.Column('name_hash', sa.String(40),
+                      nullable=False, server_default='')
         )
 
         schedulers.create()

master/buildbot/test/unit/test_util.py

@@ -311,26 +311,26 @@ class StringToBoolean(unittest.TestCase):
 
     def test_it(self):
         stringValues = [
-            ('on', True),
-            ('true', True),
-            ('yes', True),
-            ('1', True),
-            ('off', False),
-            ('false', False),
-            ('no', False),
-            ('0', False),
-            ('ON', True),
-            ('TRUE', True),
-            ('YES', True),
-            ('OFF', False),
-            ('FALSE', False),
-            ('NO', False),
+            (b'on', True),
+            (b'true', True),
+            (b'yes', True),
+            (b'1', True),
+            (b'off', False),
+            (b'false', False),
+            (b'no', False),
+            (b'0', False),
+            (b'ON', True),
+            (b'TRUE', True),
+            (b'YES', True),
+            (b'OFF', False),
+            (b'FALSE', False),
+            (b'NO', False),
         ]
         for s, b in stringValues:
             self.assertEqual(util.string2boolean(s), b, repr(s))
 
     def test_ascii(self):
-        rv = util.ascii2unicode('abcd')
+        rv = util.ascii2unicode(b'abcd')
         self.assertEqual((rv, type(rv)), (u'abcd', text_type))
 
     def test_nonascii(self):

master/buildbot/test/unit/test_www_auth.py

@@ -48,13 +48,13 @@ def setUp(self):
     def test_getChild_login(self):
         glr = mock.Mock(name='glr')
         self.master.www.auth.getLoginResource = glr
-        child = self.rsrc.getChild('login', mock.Mock(name='req'))
+        child = self.rsrc.getChild(b'login', mock.Mock(name='req'))
         self.assertIdentical(child, glr())
 
     def test_getChild_logout(self):
         glr = mock.Mock(name='glr')
         self.master.www.auth.getLogoutResource = glr
-        child = self.rsrc.getChild('logout', mock.Mock(name='req'))
+        child = self.rsrc.getChild(b'logout', mock.Mock(name='req'))
         self.assertIdentical(child, glr())
 
 
@@ -64,7 +64,7 @@ def setUp(self):
         self.auth = auth.AuthBase()
         self.master = self.make_master(url='h:/a/b/')
         self.auth.master = self.master
-        self.req = self.make_request('/')
+        self.req = self.make_request(b'/')
 
     @defer.inlineCallbacks
     def test_maybeAutoLogin(self):
@@ -106,7 +106,7 @@ class RemoteUserAuth(www.WwwTestMixin, unittest.TestCase):
     def setUp(self):
         self.auth = auth.RemoteUserAuth(header='HDR')
         self.make_master()
-        self.request = self.make_request('/')
+        self.request = self.make_request(b'/')
 
     @defer.inlineCallbacks
     def test_maybeAutoLogin(self):
@@ -172,9 +172,9 @@ def test_render(self):
         self.setUpAuthResource()
         self.rsrc = auth.LoginResource(self.master)
         self.rsrc.renderLogin = mock.Mock(
-            spec=self.rsrc.renderLogin, return_value=defer.succeed('hi'))
+            spec=self.rsrc.renderLogin, return_value=defer.succeed(b'hi'))
 
-        yield self.render_resource(self.rsrc, '/auth/login')
+        yield self.render_resource(self.rsrc, b'/auth/login')
         self.rsrc.renderLogin.assert_called_with(mock.ANY)
 
 
@@ -196,8 +196,8 @@ def updateUserInfo(request):
 
         self.auth.updateUserInfo = mock.Mock(side_effect=updateUserInfo)
 
-        res = yield self.render_resource(self.rsrc, '/auth/login')
-        self.assertEqual(res, "")
+        res = yield self.render_resource(self.rsrc, b'/auth/login')
+        self.assertEqual(res, b'')
         self.assertFalse(self.auth.maybeAutoLogin.called)
         self.auth.updateUserInfo.assert_called_with(mock.ANY)
         self.assertEqual(self.master.session.user_info,
@@ -213,6 +213,6 @@ def setUp(self):
     @defer.inlineCallbacks
     def test_render(self):
         self.master.session.expire = mock.Mock()
-        res = yield self.render_resource(self.rsrc, '/auth/logout')
-        self.assertEqual(res, "")
+        res = yield self.render_resource(self.rsrc, b'/auth/logout')
+        self.assertEqual(res, b'')
         self.master.session.expire.assert_called_with()