diff --git a/.py3.notworking.txt b/.py3.notworking.txt index 7a277ff1c09..f861bb5673e 100644 --- a/.py3.notworking.txt +++ b/.py3.notworking.txt @@ -1,6 +1,5 @@ buildbot.test.integration.test_URLs.UrlForBuildMaster.test_url buildbot.test.integration.test_trigger.TriggeringMaster.test_trigger -buildbot.test.integration.test_www.Www.test_masters buildbot.test.unit.test_data_properties.Properties.test_setBuildProperties buildbot.test.unit.test_reporters_words.TestContactChannel.test_command_watch_builder0_get_notifications buildbot.test.unit.test_schedulers_dependent.Dependent.test_related_buildset_FAILURE @@ -9,44 +8,4 @@ buildbot.test.unit.test_schedulers_dependent.Dependent.test_related_buildset_WAR buildbot.test.unit.test_schedulers_dependent.Dependent.test_unrelated_buildset buildbot.test.unit.test_schedulers_triggerable.Triggerable.test_trigger buildbot.test.unit.test_schedulers_triggerable.Triggerable.test_trigger_overlapping -buildbot.test.unit.test_schedulers_trysched.Try_Jobdir.test_parseJob_v5 -buildbot.test.unit.test_schedulers_trysched.Try_Jobdir.test_parseJob_v5_invalid_json -buildbot.test.unit.test_schedulers_trysched.Try_Jobdir.test_parseJob_v5_no_builders -buildbot.test.unit.test_schedulers_trysched.Try_Jobdir.test_parseJob_v5_no_properties buildbot.test.unit.test_www_config.IndexResource.test_render -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_fields -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_filter_and_order -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_filter_and_order_desc -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_filter_pagination -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_limit -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_limit_at_end -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_limit_past_end -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_list_filter -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_offset -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_offset_limit -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_offset_past_end -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_operator_filter -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_order -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_order_desc -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_collection_simple_filter -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_api_details_none -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_decode_result_spec_not_a_collection_properties -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_decode_result_spec_properties -buildbot.test.unit.test_www_rest.V2RootResource_REST.test_raw -buildbot.test.unit.test_www_service.Test.test_reconfigService_expiration_time -buildbot.test.unit.test_www_service.Test.test_reconfigService_port -buildbot.test.unit.test_www_service.Test.test_reconfigService_port_changes -buildbot.test.unit.test_www_service.Test.test_reconfigService_port_changes_to_none -buildbot.test.unit.test_www_service.Test.test_reconfigService_reconfigResources -buildbot.test.unit.test_www_service.Test.test_setupSiteWithHook -buildbot.test.unit.test_www_service.Test.test_setupSiteWithHookAndAuth -buildbot.test.unit.test_www_service.TestBuildbotSite.test_getSession_from_bad_jwt -buildbot.test.unit.test_www_service.TestBuildbotSite.test_getSession_from_correct_jwt -buildbot.test.unit.test_www_service.TestBuildbotSite.test_getSession_from_expired_jwt -buildbot.test.unit.test_www_service.TestBuildbotSite.test_getSession_with_no_user_info -buildbot.test.unit.test_www_service.TestBuildbotSite.test_updateSession -buildbot.test.unit.test_www_sse.EventResource.test_listen -buildbot.test.unit.test_www_sse.EventResource.test_listen_add_then_close -buildbot.test.unit.test_www_sse.EventResource.test_listen_add_then_remove -buildbot.test.unit.test_www_sse.EventResource.test_simpleapi diff --git a/master/buildbot/data/resultspec.py b/master/buildbot/data/resultspec.py index 1619083bb31..d5445646323 100644 --- a/master/buildbot/data/resultspec.py +++ b/master/buildbot/data/resultspec.py @@ -173,9 +173,10 @@ def __init__(self, filters=None, fields=None, properties=None, order=None, self.fieldMapping = {} def __repr__(self): - return "ResultSpec(**" + repr(dict(filters=self.filters, fields=self.fields, - properties=self.properties, - order=self.order, limit=self.limit, offset=self.offset)) + ")" + return ("ResultSpec(**{{'filters': {}, 'fields': {}, 'properties': {}, " + "'order': {}, 'limit': {}, 'offset': {}").format( + self.filters, self.fields, self.properties, self.order, + self.limit, self.offset) + "})" def __eq__(self, b): for i in ['filters', 'fields', 'properties', 'order', 'limit', 'offset']: diff --git a/master/buildbot/db/migrate/versions/049_add_schedulers_enabled.py b/master/buildbot/db/migrate/versions/049_add_schedulers_enabled.py index 9cb56da4793..9ab6e77806f 100644 --- a/master/buildbot/db/migrate/versions/049_add_schedulers_enabled.py +++ b/master/buildbot/db/migrate/versions/049_add_schedulers_enabled.py @@ -17,6 +17,7 @@ from __future__ import print_function import sqlalchemy as sa + from buildbot.util import sautils @@ -24,5 +25,6 @@ def upgrade(migrate_engine): metadata = sa.MetaData() metadata.bind = migrate_engine schedulers_table = sautils.Table('schedulers', metadata, autoload=True) - enabled = sa.Column('enabled', sa.SmallInteger, nullable=False, server_default="1") + enabled = sa.Column('enabled', sa.SmallInteger, + nullable=False, server_default="1") enabled.create(schedulers_table) diff --git a/master/buildbot/db/pool.py b/master/buildbot/db/pool.py index 4a6beb9c3b4..6955eebaa0e 100644 --- a/master/buildbot/db/pool.py +++ b/master/buildbot/db/pool.py @@ -32,7 +32,6 @@ from buildbot.db.schedulers import SchedulerAlreadyClaimedError from buildbot.process import metrics - # set this to True for *very* verbose query debugging output; this can # be monkey-patched from master.cfg, too: # from buildbot.db import pool @@ -146,7 +145,8 @@ def _start(self): def _stop(self): self._stop_evt = None - threads.deferToThreadPool(self.reactor, self._pool, self.engine.dispose) + threads.deferToThreadPool( + self.reactor, self._pool, self.engine.dispose) self._pool.stop() self.running = False @@ -201,7 +201,8 @@ def __thd(self, with_engine, callable, args, kwargs): # and re-try log.err(e, 'retrying {} after sql error {}'.format(callable, e)) continue - # AlreadyClaimedError are normal especially in a multimaster configuration + # AlreadyClaimedError are normal especially in a multimaster + # configuration except (AlreadyClaimedError, ChangeSourceAlreadyClaimedError, SchedulerAlreadyClaimedError): raise except Exception as e: diff --git a/master/buildbot/schedulers/trysched.py b/master/buildbot/schedulers/trysched.py index ead4d9746c3..7df07a16932 100644 --- a/master/buildbot/schedulers/trysched.py +++ b/master/buildbot/schedulers/trysched.py @@ -195,7 +195,8 @@ def postprocess_parsed_job(): postprocess_parsed_job() elif ver == "5": try: - parsed_job = json.loads(p.strings[0]) + data = bytes2NativeString(p.strings[0]) + parsed_job = json.loads(data) except ValueError: raise BadJobfile("unable to parse JSON") postprocess_parsed_job() diff --git a/master/buildbot/test/fake/fakedb.py b/master/buildbot/test/fake/fakedb.py index 62e2be2cb3c..f7d8e675422 100644 --- a/master/buildbot/test/fake/fakedb.py +++ b/master/buildbot/test/fake/fakedb.py @@ -38,6 +38,7 @@ from buildbot.db import changesources from buildbot.db import schedulers from buildbot.test.util import validation +from buildbot.util import bytes2NativeString from buildbot.util import datetime2epoch from buildbot.util import service from buildbot.util import unicode2bytes @@ -1655,7 +1656,7 @@ def setState(self, objectid, name, value): def atomicCreateState(self, objectid, name, thd_create_callback): value = thd_create_callback() - self.states[objectid][name] = json.dumps(value) + self.states[objectid][name] = json.dumps(bytes2NativeString(value)) return defer.succeed(value) # fake methods diff --git a/master/buildbot/test/fake/fakemq.py b/master/buildbot/test/fake/fakemq.py index 4a5a5517a87..8cc1ad65991 100644 --- a/master/buildbot/test/fake/fakemq.py +++ b/master/buildbot/test/fake/fakemq.py @@ -15,6 +15,7 @@ from __future__ import absolute_import from __future__ import print_function +from future.utils import string_types from twisted.internet import defer @@ -52,7 +53,10 @@ def produce(self, routingKey, data): # routing key # if self.verifyMessages: # validation.verifyMessage(self.testcase, routingKey, data) - if any(not isinstance(k, str) for k in routingKey): + + # TODO: the key to a mq should be changed so that it is unicode + # (text_type). + if any(not isinstance(k, (bytes, string_types)) for k in routingKey): raise AssertionError("%s is not all strings" % (routingKey,)) self.productions.append((routingKey, data)) # note - no consumers are called: IT'S A FAKE @@ -69,7 +73,8 @@ def callConsumer(self, routingKey, msg): raise AssertionError("no consumer found") def startConsuming(self, callback, filter, persistent_name=None): - if any(not isinstance(k, str) and k is not None for k in filter): + if any(not isinstance(k, (bytes, string_types)) and + k is not None for k in filter): raise AssertionError("%s is not a filter" % (filter,)) qref = FakeQueueRef() qref.qrefs = self.qrefs diff --git a/master/buildbot/test/integration/test_www.py b/master/buildbot/test/integration/test_www.py index 20e6b444b4d..4db0b6bedb8 100644 --- a/master/buildbot/test/integration/test_www.py +++ b/master/buildbot/test/integration/test_www.py @@ -33,6 +33,8 @@ from buildbot.test.fake import fakemaster from buildbot.test.util import db from buildbot.test.util import www +from buildbot.util import bytes2NativeString +from buildbot.util import unicode2bytes from buildbot.www import service as wwwservice from buildbot.www import auth from buildbot.www import authz @@ -54,7 +56,7 @@ def dataReceived(self, bytes): def connectionLost(self, reason): if reason.check(client.ResponseDone): - self.finishedDeferred.callback(''.join(self.body)) + self.finishedDeferred.callback(b''.join(self.body)) else: self.finishedDeferred.errback(reason) @@ -103,6 +105,7 @@ def setUp(self): # the config. The second reconfig isn't really required, but doesn't # hurt. self.url = 'http://127.0.0.1:%d/' % master.www.getPortnum() + self.url = unicode2bytes(self.url) master.config.buildbotURL = self.url yield master.www.reconfigServiceWithBuildbotConfig(master.config) @@ -126,7 +129,7 @@ def tearDown(self): @defer.inlineCallbacks def apiGet(self, url, expect200=True): - pg = yield self.agent.request('GET', url) + pg = yield self.agent.request(b'GET', url) # this is kind of obscene, but protocols are like that d = defer.Deferred() @@ -139,10 +142,10 @@ def apiGet(self, url, expect200=True): if expect200 and pg.code != 200: self.fail("did not get 200 response for '%s'" % (url,)) - defer.returnValue(json.loads(body)) + defer.returnValue(json.loads(bytes2NativeString(body))) def link(self, suffix): - return self.url + 'api/v2/' + suffix + return self.url + b'api/v2/' + suffix # tests @@ -159,7 +162,7 @@ def test_masters(self): active=1, last_active=OTHERTIME), ]) - res = yield self.apiGet(self.link('masters')) + res = yield self.apiGet(self.link(b'masters')) self.assertEqual(res, { 'masters': [ {'active': False, 'masterid': 7, 'name': 'some:master', @@ -171,7 +174,7 @@ def test_masters(self): 'total': 2, }}) - res = yield self.apiGet(self.link('masters/7')) + res = yield self.apiGet(self.link(b'masters/7')) self.assertEqual(res, { 'masters': [ {'active': False, 'masterid': 7, 'name': 'some:master', diff --git a/master/buildbot/test/unit/test_data_types.py b/master/buildbot/test/unit/test_data_types.py index 9d52ac8a22a..40251ad55ce 100644 --- a/master/buildbot/test/unit/test_data_types.py +++ b/master/buildbot/test/unit/test_data_types.py @@ -108,24 +108,24 @@ class Boolean(TypeMixin, unittest.TestCase): good = [True, False] bad = [None, 0, 1] stringValues = [ - ('on', True), - ('true', True), - ('yes', True), - ('1', True), - ('off', False), - ('false', False), - ('no', False), - ('0', False), - ('ON', True), - ('TRUE', True), - ('YES', True), - ('OFF', False), - ('FALSE', False), - ('NO', False), + (b'on', True), + (b'true', True), + (b'yes', True), + (b'1', True), + (b'off', False), + (b'false', False), + (b'no', False), + (b'0', False), + (b'ON', True), + (b'TRUE', True), + (b'YES', True), + (b'OFF', False), + (b'FALSE', False), + (b'NO', False), ] cmpResults = [ - (False, 'no', 0), - (True, 'true', 0), + (False, b'no', 0), + (True, b'true', 0), ] diff --git a/master/buildbot/test/unit/test_db_migrate_versions_049_add_schedulers_enabled.py b/master/buildbot/test/unit/test_db_migrate_versions_049_add_schedulers_enabled.py index b625d4e1115..25fd24f4423 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_049_add_schedulers_enabled.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_049_add_schedulers_enabled.py @@ -17,6 +17,7 @@ from __future__ import print_function import sqlalchemy as sa + from twisted.trial import unittest from buildbot.test.util import migration @@ -42,7 +43,8 @@ def create_tables_thd(self, conn): # name for this scheduler, as given in the configuration, plus a hash # of that name used for a unique index sa.Column('name', sa.Text, nullable=False), - sa.Column('name_hash', sa.String(40), nullable=False, server_default='') + sa.Column('name_hash', sa.String(40), + nullable=False, server_default='') ) schedulers.create() diff --git a/master/buildbot/test/unit/test_util.py b/master/buildbot/test/unit/test_util.py index d1c0495b5cf..4704df899c9 100644 --- a/master/buildbot/test/unit/test_util.py +++ b/master/buildbot/test/unit/test_util.py @@ -311,26 +311,26 @@ class StringToBoolean(unittest.TestCase): def test_it(self): stringValues = [ - ('on', True), - ('true', True), - ('yes', True), - ('1', True), - ('off', False), - ('false', False), - ('no', False), - ('0', False), - ('ON', True), - ('TRUE', True), - ('YES', True), - ('OFF', False), - ('FALSE', False), - ('NO', False), + (b'on', True), + (b'true', True), + (b'yes', True), + (b'1', True), + (b'off', False), + (b'false', False), + (b'no', False), + (b'0', False), + (b'ON', True), + (b'TRUE', True), + (b'YES', True), + (b'OFF', False), + (b'FALSE', False), + (b'NO', False), ] for s, b in stringValues: self.assertEqual(util.string2boolean(s), b, repr(s)) def test_ascii(self): - rv = util.ascii2unicode('abcd') + rv = util.ascii2unicode(b'abcd') self.assertEqual((rv, type(rv)), (u'abcd', text_type)) def test_nonascii(self): diff --git a/master/buildbot/test/unit/test_www_auth.py b/master/buildbot/test/unit/test_www_auth.py index 49583e2b1e5..ed3b8b41e4a 100644 --- a/master/buildbot/test/unit/test_www_auth.py +++ b/master/buildbot/test/unit/test_www_auth.py @@ -48,13 +48,13 @@ def setUp(self): def test_getChild_login(self): glr = mock.Mock(name='glr') self.master.www.auth.getLoginResource = glr - child = self.rsrc.getChild('login', mock.Mock(name='req')) + child = self.rsrc.getChild(b'login', mock.Mock(name='req')) self.assertIdentical(child, glr()) def test_getChild_logout(self): glr = mock.Mock(name='glr') self.master.www.auth.getLogoutResource = glr - child = self.rsrc.getChild('logout', mock.Mock(name='req')) + child = self.rsrc.getChild(b'logout', mock.Mock(name='req')) self.assertIdentical(child, glr()) @@ -64,7 +64,7 @@ def setUp(self): self.auth = auth.AuthBase() self.master = self.make_master(url='h:/a/b/') self.auth.master = self.master - self.req = self.make_request('/') + self.req = self.make_request(b'/') @defer.inlineCallbacks def test_maybeAutoLogin(self): @@ -106,7 +106,7 @@ class RemoteUserAuth(www.WwwTestMixin, unittest.TestCase): def setUp(self): self.auth = auth.RemoteUserAuth(header='HDR') self.make_master() - self.request = self.make_request('/') + self.request = self.make_request(b'/') @defer.inlineCallbacks def test_maybeAutoLogin(self): @@ -172,9 +172,9 @@ def test_render(self): self.setUpAuthResource() self.rsrc = auth.LoginResource(self.master) self.rsrc.renderLogin = mock.Mock( - spec=self.rsrc.renderLogin, return_value=defer.succeed('hi')) + spec=self.rsrc.renderLogin, return_value=defer.succeed(b'hi')) - yield self.render_resource(self.rsrc, '/auth/login') + yield self.render_resource(self.rsrc, b'/auth/login') self.rsrc.renderLogin.assert_called_with(mock.ANY) @@ -196,8 +196,8 @@ def updateUserInfo(request): self.auth.updateUserInfo = mock.Mock(side_effect=updateUserInfo) - res = yield self.render_resource(self.rsrc, '/auth/login') - self.assertEqual(res, "") + res = yield self.render_resource(self.rsrc, b'/auth/login') + self.assertEqual(res, b'') self.assertFalse(self.auth.maybeAutoLogin.called) self.auth.updateUserInfo.assert_called_with(mock.ANY) self.assertEqual(self.master.session.user_info, @@ -213,6 +213,6 @@ def setUp(self): @defer.inlineCallbacks def test_render(self): self.master.session.expire = mock.Mock() - res = yield self.render_resource(self.rsrc, '/auth/logout') - self.assertEqual(res, "") + res = yield self.render_resource(self.rsrc, b'/auth/logout') + self.assertEqual(res, b'') self.master.session.expire.assert_called_with() diff --git a/master/buildbot/test/unit/test_www_avatar.py b/master/buildbot/test/unit/test_www_avatar.py index 1ee608aa5bc..c7a40b366b9 100644 --- a/master/buildbot/test/unit/test_www_avatar.py +++ b/master/buildbot/test/unit/test_www_avatar.py @@ -33,7 +33,7 @@ def test_default(self): rsrc = avatar.AvatarResource(master) rsrc.reconfigResource(master.config) - res = yield self.render_resource(rsrc, '/') + res = yield self.render_resource(rsrc, b'/') self.assertEqual( res, dict(redirected=avatar.AvatarResource.defaultAvatarUrl)) @@ -44,7 +44,7 @@ def test_gravatar(self): rsrc = avatar.AvatarResource(master) rsrc.reconfigResource(master.config) - res = yield self.render_resource(rsrc, '/?email=foo') + res = yield self.render_resource(rsrc, b'/?email=foo') self.assertEqual(res, dict(redirected='//www.gravatar.com/avatar/acbd18db4cc2f85ce' 'def654fccc4a4d8?d=retro&s=32')) @@ -53,15 +53,17 @@ def test_custom(self): class CustomAvatar(avatar.AvatarBase): def getUserAvatar(self, email, size, defaultAvatarUrl): - return defer.succeed(("image/png", email + str(size) + defaultAvatarUrl)) + return defer.succeed((b"image/png", email + + str(size).encode('utf-8') + + defaultAvatarUrl)) master = self.make_master( url='http://a/b/', auth=auth.NoAuth(), avatar_methods=[CustomAvatar()]) rsrc = avatar.AvatarResource(master) rsrc.reconfigResource(master.config) - res = yield self.render_resource(rsrc, '/?email=foo') - self.assertEqual(res, "foo32http://a/b/img/nobody.png") + res = yield self.render_resource(rsrc, b'/?email=foo') + self.assertEqual(res, b"foo32http://a/b/img/nobody.png") @defer.inlineCallbacks def test_custom_not_found(self): @@ -71,11 +73,11 @@ class CustomAvatar(avatar.AvatarBase): def getUserAvatar(self, email, size, defaultAvatarUrl): return defer.succeed(None) - master = self.make_master(url='http://a/b/', auth=auth.NoAuth(), + master = self.make_master(url=b'http://a/b/', auth=auth.NoAuth(), avatar_methods=[CustomAvatar(), avatar.AvatarGravatar()]) rsrc = avatar.AvatarResource(master) rsrc.reconfigResource(master.config) - res = yield self.render_resource(rsrc, '/?email=foo') + res = yield self.render_resource(rsrc, b'/?email=foo') self.assertEqual(res, dict(redirected='//www.gravatar.com/avatar/acbd18db4cc2f85ce' 'def654fccc4a4d8?d=retro&s=32')) diff --git a/master/buildbot/test/unit/test_www_config.py b/master/buildbot/test/unit/test_www_config.py index 27b19ada8a3..e6d9b3b93e0 100644 --- a/master/buildbot/test/unit/test_www_config.py +++ b/master/buildbot/test/unit/test_www_config.py @@ -51,14 +51,14 @@ def test_render(self): vjson = json.dumps(rsrc.getEnvironmentVersions() + custom_versions) - res = yield self.render_resource(rsrc, '/') + res = yield self.render_resource(rsrc, b'/') _auth.maybeAutoLogin.assert_called_with(mock.ANY) exp = '{"authz": {}, "titleURL": "http://buildbot.net", "versions": %s, "title": "Buildbot", "auth": {"name": "NoAuth"}, "user": {"anonymous": true}, "buildbotURL": "h:/a/b/", "multiMaster": false, "port": null}' exp = exp % vjson self.assertEqual(res, exp) master.session.user_info = dict(name="me", email="me@me.org") - res = yield self.render_resource(rsrc, '/') + res = yield self.render_resource(rsrc, b'/') exp = '{"authz": {}, "titleURL": "http://buildbot.net", "versions": %s, "title": "Buildbot", "auth": {"name": "NoAuth"}, "user": {"email": "me@me.org", "name": "me"}, "buildbotURL": "h:/a/b/", "multiMaster": false, "port": null}' exp = exp % vjson self.assertEqual(res, exp) @@ -66,7 +66,7 @@ def test_render(self): master = self.make_master( url='h:/a/c/', auth=_auth, versions=custom_versions) rsrc.reconfigResource(master.config) - res = yield self.render_resource(rsrc, '/') + res = yield self.render_resource(rsrc, b'/') exp = '{"authz": {}, "titleURL": "http://buildbot.net", "versions": %s, "title": "Buildbot", "auth": {"name": "NoAuth"}, "user": {"anonymous": true}, "buildbotURL": "h:/a/b/", "multiMaster": false, "port": null}' exp = exp % vjson self.assertIn(res, exp) diff --git a/master/buildbot/test/unit/test_www_oauth.py b/master/buildbot/test/unit/test_www_oauth.py index 035b0ea174a..2dac11eb352 100644 --- a/master/buildbot/test/unit/test_www_oauth.py +++ b/master/buildbot/test/unit/test_www_oauth.py @@ -255,20 +255,20 @@ class fakeAuth(object): rsrc = self.githubAuth.getLoginResource() rsrc.auth = fakeAuth() - res = yield self.render_resource(rsrc, '/') + res = yield self.render_resource(rsrc, b'/') rsrc.auth.getLoginURL.assert_called_once_with(None) rsrc.auth.verifyCode.assert_not_called() self.assertEqual(res, {'redirected': '://'}) rsrc.auth.getLoginURL.reset_mock() rsrc.auth.verifyCode.reset_mock() - res = yield self.render_resource(rsrc, '/?code=code!') + res = yield self.render_resource(rsrc, b'/?code=code!') rsrc.auth.getLoginURL.assert_not_called() - rsrc.auth.verifyCode.assert_called_once_with("code!") + rsrc.auth.verifyCode.assert_called_once_with(b"code!") self.assertEqual(self.master.session.user_info, {'username': 'bar'}) self.assertEqual(res, {'redirected': '://me'}) - res = yield self.render_resource(rsrc, '/?token=token!') + res = yield self.render_resource(rsrc, b'/?token=token!') rsrc.auth.getLoginURL.assert_not_called() - rsrc.auth.acceptToken.assert_called_once_with("token!") + rsrc.auth.acceptToken.assert_called_once_with(b"token!") self.assertEqual(self.master.session.user_info, {'username': 'bar'}) self.assertEqual(res, {'redirected': '://me'}) @@ -350,7 +350,8 @@ class HomePage(Resource): def render_GET(self, request): info = request.getSession().user_info reactor.callLater(0, d.callback, info) - return "WORKED: %s" % (info) + return (b"WORKED: " + + info + b"") class MySite(Site): @@ -359,10 +360,10 @@ def makeSession(self): session = self.sessions[uid] = self.sessionFactory(self, uid) return session root = Resource() - root.putChild("", HomePage()) + root.putChild(b"", HomePage()) auth = Resource() - root.putChild('auth', auth) - auth.putChild('login', self.auth.getLoginResource()) + root.putChild(b'auth', auth) + auth.putChild(b'login', self.auth.getLoginResource()) site = MySite(root) l = reactor.listenTCP(5000, site) diff --git a/master/buildbot/test/unit/test_www_resource.py b/master/buildbot/test/unit/test_www_resource.py index 9fe3e95a2b5..cfe0d3e5261 100644 --- a/master/buildbot/test/unit/test_www_resource.py +++ b/master/buildbot/test/unit/test_www_resource.py @@ -30,12 +30,12 @@ class ResourceSubclass(resource.Resource): class Resource(www.WwwTestMixin, unittest.TestCase): def test_base_url(self): - master = self.make_master(url='h:/a/b/') + master = self.make_master(url=b'h:/a/b/') rsrc = resource.Resource(master) - self.assertEqual(rsrc.base_url, 'h:/a/b/') + self.assertEqual(rsrc.base_url, b'h:/a/b/') def test_reconfigResource_registration(self): - master = self.make_master(url='h:/a/b/') + master = self.make_master(url=b'h:/a/b/') rsrc = ResourceSubclass(master) master.www.resourceNeedsReconfigs.assert_called_with(rsrc) @@ -43,7 +43,7 @@ def test_reconfigResource_registration(self): class RedirectResource(www.WwwTestMixin, unittest.TestCase): def test_redirect(self): - master = self.make_master(url='h:/a/b/') - rsrc = resource.RedirectResource(master, 'foo') - self.render_resource(rsrc, '/') - self.assertEqual(self.request.redirected_to, 'h:/a/b/foo') + master = self.make_master(url=b'h:/a/b/') + rsrc = resource.RedirectResource(master, b'foo') + self.render_resource(rsrc, b'/') + self.assertEqual(self.request.redirected_to, b'h:/a/b/foo') diff --git a/master/buildbot/test/unit/test_www_rest.py b/master/buildbot/test/unit/test_www_rest.py index 8a5e873ee4c..430d86c47ef 100644 --- a/master/buildbot/test/unit/test_www_rest.py +++ b/master/buildbot/test/unit/test_www_rest.py @@ -31,6 +31,8 @@ from buildbot.test.fake import endpoint from buildbot.test.util import www +from buildbot.util import bytes2NativeString +from buildbot.util import unicode2bytes from buildbot.www import authz from buildbot.www import rest from buildbot.www.rest import JSONRPC_CODES @@ -45,27 +47,30 @@ def test_render(self): master = self.make_master(url='h:/a/b/') rsrc = rest.RestRootResource(master) - d = self.render_resource(rsrc, '/') + d = self.render_resource(rsrc, b'/') @d.addCallback def check(rv): - self.assertIn('api_versions', rv) + self.assertIn(b'api_versions', rv) return d def test_versions(self): master = self.make_master(url='h:/a/b/') rsrc = rest.RestRootResource(master) - self.assertEqual(sorted(rsrc.listNames()), - sorted(['latest'] + - ['v%d' % v for v in range(2, self.maxVersion + 1)])) + versions = [unicode2bytes('v{}'.format(v)) + for v in range(2, self.maxVersion + 1)] + versions = [unicode2bytes(v) for v in versions] + versions.append(b'latest') + self.assertEqual(sorted(rsrc.listNames()), sorted(versions)) def test_versions_limited(self): master = self.make_master(url='h:/a/b/') master.config.www['rest_minimum_version'] = 2 rsrc = rest.RestRootResource(master) - self.assertEqual(sorted(rsrc.listNames()), - sorted(['latest'] + - ['v%d' % v for v in range(2, self.maxVersion + 1)])) + versions = [unicode2bytes('v{}'.format(v)) + for v in range(2, self.maxVersion + 1)] + versions.append(b'latest') + self.assertEqual(sorted(rsrc.listNames()), sorted(versions)) class V2RootResource(www.WwwTestMixin, unittest.TestCase): @@ -77,20 +82,21 @@ def setUp(self): self.rsrc.reconfigResource(self.master.config) def assertSimpleError(self, message, responseCode): - self.assertRequest(content=json.dumps({'error': message}), + content = json.dumps({'error': message}) + self.assertRequest(content=unicode2bytes(content), responseCode=responseCode) @defer.inlineCallbacks def test_failure(self): self.rsrc.renderRest = mock.Mock( return_value=defer.fail(RuntimeError('oh noes'))) - yield self.render_resource(self.rsrc, '/') + yield self.render_resource(self.rsrc, b'/') self.assertSimpleError('internal error - see logs', 500) self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1) @defer.inlineCallbacks def test_invalid_http_method(self): - yield self.render_resource(self.rsrc, '/', method='PATCH') + yield self.render_resource(self.rsrc, b'/', method=b'PATCH') self.assertSimpleError('invalid HTTP method', 400) def test_default_origin(self): @@ -121,43 +127,44 @@ def setUp(self): self.master = self.make_master(url='h:/') self.master.data._scanModule(endpoint) self.rsrc = rest.V2RootResource(self.master) - self.master.config.www['allowed_origins'] = ['h://good'] + self.master.config.www['allowed_origins'] = [b'h://good'] self.rsrc.reconfigResource(self.master.config) def renderRest(request): - request.write('ok') + request.write(b'ok') return defer.succeed(None) self.rsrc.renderRest = renderRest - def assertOk(self, expectHeaders=True, content='ok', origin='h://good'): + def assertOk(self, expectHeaders=True, content=b'ok', origin=b'h://good'): hdrs = { - 'access-control-allow-origin': [origin], - 'access-control-allow-headers': ['Content-Type'], - 'access-control-max-age': ['3600'], + b'access-control-allow-origin': [origin], + b'access-control-allow-headers': [b'Content-Type'], + b'access-control-max-age': [b'3600'], } if expectHeaders else {} self.assertRequest(content=content, responseCode=200, headers=hdrs) def assertNotOk(self, message): - self.assertRequest(content=json.dumps({'error': message}), - responseCode=400) + content = json.dumps({'error': message}) + content = unicode2bytes(content) + self.assertRequest(content=content, responseCode=400) @defer.inlineCallbacks def test_cors_no_origin(self): # if the browser doesn't send Origin, there's nothing we can do to # protect the user - yield self.render_resource(self.rsrc, '/') + yield self.render_resource(self.rsrc, b'/') self.assertOk(expectHeaders=False) @defer.inlineCallbacks def test_cors_origin_match(self): - yield self.render_resource(self.rsrc, '/', origin='h://good') + yield self.render_resource(self.rsrc, b'/', origin=b'h://good') self.assertOk() @defer.inlineCallbacks def test_cors_origin_match_star(self): self.master.config.www['allowed_origins'] = ['*'] self.rsrc.reconfigResource(self.master.config) - yield self.render_resource(self.rsrc, '/', origin='h://good') + yield self.render_resource(self.rsrc, b'/', origin=b'h://good') self.assertOk() @defer.inlineCallbacks @@ -165,50 +172,51 @@ def test_cors_origin_patterns(self): self.master.config.www['allowed_origins'] = ['h://*.good', 'hs://*.secure'] self.rsrc.reconfigResource(self.master.config) - yield self.render_resource(self.rsrc, '/', origin='h://foo.good') - self.assertOk(origin='h://foo.good') - yield self.render_resource(self.rsrc, '/', origin='hs://x.secure') - self.assertOk(origin='hs://x.secure') - yield self.render_resource(self.rsrc, '/', origin='h://x.secure') + yield self.render_resource(self.rsrc, b'/', origin=b'h://foo.good') + self.assertOk(origin=b'h://foo.good') + yield self.render_resource(self.rsrc, b'/', origin=b'hs://x.secure') + self.assertOk(origin=b'hs://x.secure') + yield self.render_resource(self.rsrc, b'/', origin=b'h://x.secure') self.assertNotOk('invalid origin') @defer.inlineCallbacks def test_cors_origin_mismatch(self): - yield self.render_resource(self.rsrc, '/', origin='h://bad') + yield self.render_resource(self.rsrc, b'/', origin=b'h://bad') self.assertNotOk('invalid origin') @defer.inlineCallbacks def test_cors_origin_mismatch_post(self): - yield self.render_resource(self.rsrc, '/', method='POST', origin='h://bad') - self.assertRequest(content=json.dumps({'error': {'message': 'invalid origin'}}), - responseCode=400) + yield self.render_resource(self.rsrc, b'/', method=b'POST', origin=b'h://bad') + content = json.dumps({'error': {'message': 'invalid origin'}}) + content = unicode2bytes(content) + self.assertRequest(content=content, responseCode=400) @defer.inlineCallbacks def test_cors_origin_preflight_match_GET(self): - yield self.render_resource(self.rsrc, '/', - method='OPTIONS', origin='h://good', - access_control_request_method='GET') - self.assertOk(content='') + yield self.render_resource(self.rsrc, b'/', + method=b'OPTIONS', origin=b'h://good', + access_control_request_method=b'GET') + self.assertOk(content=b'') @defer.inlineCallbacks def test_cors_origin_preflight_match_POST(self): - yield self.render_resource(self.rsrc, '/', - method='OPTIONS', origin='h://good', - access_control_request_method='POST') - self.assertOk(content='') + yield self.render_resource(self.rsrc, b'/', + method=b'OPTIONS', origin=b'h://good', + access_control_request_method=b'POST') + self.assertOk(content=b'') @defer.inlineCallbacks def test_cors_origin_preflight_bad_method(self): - yield self.render_resource(self.rsrc, '/', - method='OPTIONS', origin='h://good', - access_control_request_method='PATCH') + yield self.render_resource(self.rsrc, b'/', + method=b'OPTIONS', origin=b'h://good', + access_control_request_method=b'PATCH') self.assertNotOk(message='invalid method') @defer.inlineCallbacks def test_cors_origin_preflight_bad_origin(self): - yield self.render_resource(self.rsrc, '/', - method='OPTIONS', origin='h://bad', - access_control_request_method='GET') + yield self.render_resource(self.rsrc, b'/', + method=b'OPTIONS', origin=b'h://bad', + access_control_request_method=b'GET') self.assertNotOk(message='invalid origin') @@ -234,8 +242,8 @@ def assertRestCollection(self, typeName, items, total=None, contentType=None, orderSignificant=False): self.assertFalse(isinstance(self.request.written, text_type)) got = {} - got['content'] = json.loads(self.request.written) - got['contentType'] = self.request.headers['content-type'] + got['content'] = json.loads(bytes2NativeString(self.request.written)) + got['contentType'] = self.request.headers[b'content-type'] got['responseCode'] = self.request.responseCode meta = {} @@ -244,14 +252,14 @@ def assertRestCollection(self, typeName, items, exp = {} exp['content'] = {typeName: items, 'meta': meta} - exp['contentType'] = [contentType or 'text/plain; charset=utf-8'] + exp['contentType'] = [contentType or b'text/plain; charset=utf-8'] exp['responseCode'] = 200 # if order is not significant, sort so the comparison works if not orderSignificant: if 'content' in got and typeName in got['content']: - got['content'][typeName].sort() - exp['content'][typeName].sort() + got['content'][typeName].sort(key=lambda x: list(x.values())) + exp['content'][typeName].sort(key=lambda x: list(x.values())) if 'meta' in got['content'] and 'links' in got['content']['meta']: got['content']['meta']['links'].sort( key=lambda l: (l['rel'], l['href'])) @@ -261,8 +269,8 @@ def assertRestCollection(self, typeName, items, def assertRestDetails(self, typeName, item, contentType=None): got = {} - got['content'] = json.loads(self.request.written) - got['contentType'] = self.request.headers['content-type'] + got['content'] = json.loads(bytes2NativeString(self.request.written)) + got['contentType'] = self.request.headers[b'content-type'] got['responseCode'] = self.request.responseCode exp = {} @@ -270,14 +278,14 @@ def assertRestDetails(self, typeName, item, typeName: [item], 'meta': {}, } - exp['contentType'] = [contentType or 'text/plain; charset=utf-8'] + exp['contentType'] = [contentType or b'text/plain; charset=utf-8'] exp['responseCode'] = 200 self.assertEqual(got, exp) def assertRestError(self, responseCode, message): got = {} - got['content'] = json.loads(self.request.written) + got['content'] = json.loads(bytes2NativeString(self.request.written)) got['responseCode'] = self.request.responseCode exp = {} @@ -288,68 +296,68 @@ def assertRestError(self, responseCode, message): @defer.inlineCallbacks def test_not_found(self): - yield self.render_resource(self.rsrc, '/not/found') + yield self.render_resource(self.rsrc, b'/not/found') self.assertRequest( contentJson=dict(error='Invalid path: not/found'), - contentType='text/plain; charset=utf-8', + contentType=b'text/plain; charset=utf-8', responseCode=404) @defer.inlineCallbacks def test_invalid_query(self): - yield self.render_resource(self.rsrc, '/test?huh=1') + yield self.render_resource(self.rsrc, b'/test?huh=1') self.assertRequest( contentJson=dict(error="unrecognized query parameter 'huh'"), - contentType='text/plain; charset=utf-8', + contentType=b'text/plain; charset=utf-8', responseCode=400) @defer.inlineCallbacks def test_raw(self): - yield self.render_resource(self.rsrc, '/rawtest') + yield self.render_resource(self.rsrc, b'/rawtest') self.assertRequest( - content="value", - contentType='text/test; charset=utf-8', + content=b"value", + contentType=b'text/test; charset=utf-8', responseCode=200, - headers={"content-disposition": ['attachment; filename=test.txt']}) + headers={b"content-disposition": [b'attachment; filename=test.txt']}) @defer.inlineCallbacks def test_api_head(self): - get = yield self.render_resource(self.rsrc, '/test', method='GET') - head = yield self.render_resource(self.rsrc, '/test', method='HEAD') - self.assertEqual(head, '') - self.assertEqual(int(self.request.headers['content-length'][0]), + get = yield self.render_resource(self.rsrc, b'/test', method=b'GET') + head = yield self.render_resource(self.rsrc, b'/test', method=b'HEAD') + self.assertEqual(head, b'') + self.assertEqual(int(self.request.headers[b'content-length'][0]), len(get)) @defer.inlineCallbacks def test_api_collection(self): - yield self.render_resource(self.rsrc, '/test') + yield self.render_resource(self.rsrc, b'/test') self.assertRestCollection(typeName='tests', items=list(itervalues(endpoint.testData)), total=8) @defer.inlineCallbacks def do_test_api_collection_pagination(self, query, ids, links): - yield self.render_resource(self.rsrc, '/test' + query) + yield self.render_resource(self.rsrc, b'/test' + query) self.assertRestCollection(typeName='tests', items=[v for k, v in iteritems(endpoint.testData) if k in ids], total=8) def test_api_collection_limit(self): - return self.do_test_api_collection_pagination('?limit=2', + return self.do_test_api_collection_pagination(b'?limit=2', [13, 14], { 'self': '%(self)s?limit=2', 'next': '%(self)s?offset=2&limit=2', }) def test_api_collection_offset(self): - return self.do_test_api_collection_pagination('?offset=2', + return self.do_test_api_collection_pagination(b'?offset=2', [15, 16, 17, 18, 19, 20], { 'self': '%(self)s?offset=2', 'first': '%(self)s', }) def test_api_collection_offset_limit(self): - return self.do_test_api_collection_pagination('?offset=5&limit=2', + return self.do_test_api_collection_pagination(b'?offset=5&limit=2', [18, 19], { 'first': '%(self)s?limit=2', 'prev': '%(self)s?offset=3&limit=2', @@ -358,7 +366,7 @@ def test_api_collection_offset_limit(self): }) def test_api_collection_limit_at_end(self): - return self.do_test_api_collection_pagination('?offset=5&limit=3', + return self.do_test_api_collection_pagination(b'?offset=5&limit=3', [18, 19, 20], { 'first': '%(self)s?limit=3', 'prev': '%(self)s?offset=2&limit=3', @@ -366,7 +374,7 @@ def test_api_collection_limit_at_end(self): }) def test_api_collection_limit_past_end(self): - return self.do_test_api_collection_pagination('?offset=5&limit=20', + return self.do_test_api_collection_pagination(b'?offset=5&limit=20', [18, 19, 20], { 'first': '%(self)s?limit=20', 'prev': '%(self)s?limit=5', @@ -374,7 +382,7 @@ def test_api_collection_limit_past_end(self): }) def test_api_collection_offset_past_end(self): - return self.do_test_api_collection_pagination('?offset=50&limit=10', + return self.do_test_api_collection_pagination(b'?offset=50&limit=10', [], { 'first': '%(self)s?limit=10', 'prev': '%(self)s?offset=40&limit=10', @@ -383,39 +391,39 @@ def test_api_collection_offset_past_end(self): @defer.inlineCallbacks def test_api_collection_invalid_limit(self): - yield self.render_resource(self.rsrc, '/test?limit=foo!') + yield self.render_resource(self.rsrc, b'/test?limit=foo!') self.assertRequest( contentJson=dict(error="invalid limit"), - contentType='text/plain; charset=utf-8', + contentType=b'text/plain; charset=utf-8', responseCode=400) @defer.inlineCallbacks def test_api_collection_invalid_offset(self): - yield self.render_resource(self.rsrc, '/test?offset=foo!') + yield self.render_resource(self.rsrc, b'/test?offset=foo!') self.assertRequest( contentJson=dict(error="invalid offset"), - contentType='text/plain; charset=utf-8', + contentType=b'text/plain; charset=utf-8', responseCode=400) @defer.inlineCallbacks def test_api_collection_invalid_simple_filter_value(self): - yield self.render_resource(self.rsrc, '/test?success=sorta') + yield self.render_resource(self.rsrc, b'/test?success=sorta') self.assertRequest( contentJson=dict(error="invalid filter value for success"), - contentType='text/plain; charset=utf-8', + contentType=b'text/plain; charset=utf-8', responseCode=400) @defer.inlineCallbacks def test_api_collection_invalid_filter_value(self): - yield self.render_resource(self.rsrc, '/test?id__lt=fifteen') + yield self.render_resource(self.rsrc, b'/test?id__lt=fifteen') self.assertRequest( contentJson=dict(error="invalid filter value for id__lt"), - contentType='text/plain; charset=utf-8', + contentType=b'text/plain; charset=utf-8', responseCode=400) @defer.inlineCallbacks def test_api_collection_fields(self): - yield self.render_resource(self.rsrc, '/test?field=success&field=info') + yield self.render_resource(self.rsrc, b'/test?field=success&field=info') self.assertRestCollection(typeName='tests', items=[{'success': v['success'], 'info': v['info']} for v in itervalues(endpoint.testData)], @@ -423,15 +431,15 @@ def test_api_collection_fields(self): @defer.inlineCallbacks def test_api_collection_invalid_field(self): - yield self.render_resource(self.rsrc, '/test?field=success&field=WTF') + yield self.render_resource(self.rsrc, b'/test?field=success&field=WTF') self.assertRequest( contentJson=dict(error="no such field 'WTF'"), - contentType='text/plain; charset=utf-8', + contentType=b'text/plain; charset=utf-8', responseCode=400) @defer.inlineCallbacks def test_api_collection_simple_filter(self): - yield self.render_resource(self.rsrc, '/test?success=yes') + yield self.render_resource(self.rsrc, b'/test?success=yes') self.assertRestCollection(typeName='tests', items=[v for v in itervalues(endpoint.testData) if v['success']], @@ -439,7 +447,7 @@ def test_api_collection_simple_filter(self): @defer.inlineCallbacks def test_api_collection_list_filter(self): - yield self.render_resource(self.rsrc, '/test?tags__contains=a') + yield self.render_resource(self.rsrc, b'/test?tags__contains=a') self.assertRestCollection(typeName='tests', items=[v for v in itervalues(endpoint.testData) if 'a' in v['tags']], @@ -447,7 +455,7 @@ def test_api_collection_list_filter(self): @defer.inlineCallbacks def test_api_collection_operator_filter(self): - yield self.render_resource(self.rsrc, '/test?info__lt=skipped') + yield self.render_resource(self.rsrc, b'/test?info__lt=skipped') self.assertRestCollection(typeName='tests', items=[v for v in itervalues(endpoint.testData) if v['info'] < 'skipped'], @@ -455,7 +463,7 @@ def test_api_collection_operator_filter(self): @defer.inlineCallbacks def test_api_collection_order(self): - yield self.render_resource(self.rsrc, '/test?order=info') + yield self.render_resource(self.rsrc, b'/test?order=info') self.assertRestCollection(typeName='tests', items=sorted(list(itervalues(endpoint.testData)), key=lambda v: v['info']), @@ -463,7 +471,7 @@ def test_api_collection_order(self): @defer.inlineCallbacks def test_api_collection_filter_and_order(self): - yield self.render_resource(self.rsrc, '/test?field=info&order=info') + yield self.render_resource(self.rsrc, b'/test?field=info&order=info') self.assertRestCollection(typeName='tests', items=sorted(list([{'info': v['info']} for v in itervalues(endpoint.testData)]), @@ -472,7 +480,7 @@ def test_api_collection_filter_and_order(self): @defer.inlineCallbacks def test_api_collection_order_desc(self): - yield self.render_resource(self.rsrc, '/test?order=-info') + yield self.render_resource(self.rsrc, b'/test?order=-info') self.assertRestCollection(typeName='tests', items=sorted(list(itervalues(endpoint.testData)), key=lambda v: v['info'], reverse=True), @@ -480,7 +488,7 @@ def test_api_collection_order_desc(self): @defer.inlineCallbacks def test_api_collection_filter_and_order_desc(self): - yield self.render_resource(self.rsrc, '/test?field=info&order=-info') + yield self.render_resource(self.rsrc, b'/test?field=info&order=-info') self.assertRestCollection(typeName='tests', items=sorted(list([{'info': v['info']} for v in itervalues(endpoint.testData)]), @@ -489,19 +497,19 @@ def test_api_collection_filter_and_order_desc(self): @defer.inlineCallbacks def test_api_collection_order_on_unselected(self): - yield self.render_resource(self.rsrc, '/test?field=id&order=info') + yield self.render_resource(self.rsrc, b'/test?field=id&order=info') self.assertRestError(message="cannot order on un-selected fields", responseCode=400) @defer.inlineCallbacks def test_api_collection_filter_on_unselected(self): - yield self.render_resource(self.rsrc, '/test?field=id&info__gt=xx') + yield self.render_resource(self.rsrc, b'/test?field=id&info__gt=xx') self.assertRestError(message="cannot filter on un-selected fields", responseCode=400) @defer.inlineCallbacks def test_api_collection_filter_pagination(self): - yield self.render_resource(self.rsrc, '/test?success=false&limit=2') + yield self.render_resource(self.rsrc, b'/test?success=false&limit=2') # note that the limit/offset and total are *after* the filter self.assertRestCollection(typeName='tests', items=sorted([v for v in itervalues(endpoint.testData) @@ -510,113 +518,114 @@ def test_api_collection_filter_pagination(self): @defer.inlineCallbacks def test_api_details(self): - yield self.render_resource(self.rsrc, '/test/13') + yield self.render_resource(self.rsrc, b'/test/13') self.assertRestDetails(typeName='tests', item=endpoint.testData[13]) @defer.inlineCallbacks def test_api_details_none(self): - yield self.render_resource(self.rsrc, '/test/0') + self.maxDiff = None + yield self.render_resource(self.rsrc, b'/test/0') self.assertRequest( contentJson={u'error': u"not found while getting from endpoint for /test/n:testid with arguments" - " ResultSpec(**{'limit': None, 'filters': [], 'offset': None, " - "'fields': None, 'order': None, 'properties': []}) and {'testid': 0}"}, - contentType='text/plain; charset=utf-8', + " ResultSpec(**{'filters': [], 'fields': None, 'properties': [], " + "'order': None, 'limit': None, 'offset': None}) and {'testid': 0}"}, + contentType=b'text/plain; charset=utf-8', responseCode=404) @defer.inlineCallbacks def test_api_details_filter_fails(self): - yield self.render_resource(self.rsrc, '/test/13?success=false') + yield self.render_resource(self.rsrc, b'/test/13?success=false') self.assertRequest( contentJson=dict(error="this is not a collection"), - contentType='text/plain; charset=utf-8', + contentType=b'text/plain; charset=utf-8', responseCode=400) @defer.inlineCallbacks def test_api_details_fields(self): - yield self.render_resource(self.rsrc, '/test/13?field=info') + yield self.render_resource(self.rsrc, b'/test/13?field=info') self.assertRestDetails(typeName='tests', item={'info': endpoint.testData[13]['info']}) @defer.inlineCallbacks def test_api_with_accept(self): # when 'application/json' is accepted, the result has that type - yield self.render_resource(self.rsrc, '/test/13', - accept='application/json') + yield self.render_resource(self.rsrc, b'/test/13', + accept=b'application/json') self.assertRestDetails(typeName='tests', item=endpoint.testData[13], - contentType='application/json; charset=utf-8') + contentType=b'application/json; charset=utf-8') @defer.inlineCallbacks def test_api_fails(self): - yield self.render_resource(self.rsrc, '/test/fail') + yield self.render_resource(self.rsrc, b'/test/fail') self.assertRestError(message="RuntimeError('oh noes',)", responseCode=500) self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1) def test_decode_result_spec_raise_bad_request_on_bad_property_value(self): expected_props = [None, 'test2'] - self.make_request('/test') - self.request.args = {'property': expected_props} + self.make_request(b'/test') + self.request.args = {b'property': expected_props} self.assertRaises(BadRequest, lambda: self.rsrc.decodeResultSpec( self.request, endpoint.TestsEndpoint)) def test_decode_result_spec_limit(self): expected_limit = 5 - self.make_request('/test') - self.request.args = {'limit': str(expected_limit)} + self.make_request(b'/test') + self.request.args = {b'limit': str(expected_limit)} spec = self.rsrc.decodeResultSpec(self.request, endpoint.TestsEndpoint) self.assertEqual(spec.limit, expected_limit) def test_decode_result_spec_order(self): expected_order = 'info', - self.make_request('/test') - self.request.args = {'order': expected_order} + self.make_request(b'/test') + self.request.args = {b'order': expected_order} spec = self.rsrc.decodeResultSpec(self.request, endpoint.Test) self.assertEqual(spec.order, expected_order) def test_decode_result_spec_offset(self): expected_offset = 5 - self.make_request('/test') - self.request.args = {'offset': str(expected_offset)} + self.make_request(b'/test') + self.request.args = {b'offset': str(expected_offset)} spec = self.rsrc.decodeResultSpec(self.request, endpoint.TestsEndpoint) self.assertEqual(spec.offset, expected_offset) def test_decode_result_spec_properties(self): expected_props = ['test1', 'test2'] - self.make_request('/test') - self.request.args = {'property': expected_props} + self.make_request(b'/test') + self.request.args = {b'property': expected_props} spec = self.rsrc.decodeResultSpec(self.request, endpoint.TestsEndpoint) self.assertEqual(spec.properties[0].values, expected_props) def test_decode_result_spec_not_a_collection_limit(self): def expectRaiseBadRequest(): limit = 5 - self.make_request('/test') - self.request.args = {'limit': limit} + self.make_request(b'/test') + self.request.args = {b'limit': limit} self.rsrc.decodeResultSpec(self.request, endpoint.TestEndpoint) self.assertRaises(rest.BadRequest, expectRaiseBadRequest) def test_decode_result_spec_not_a_collection_order(self): def expectRaiseBadRequest(): order = 'info', - self.make_request('/test') - self.request.args = {'order': order} + self.make_request(b'/test') + self.request.args = {b'order': order} self.rsrc.decodeResultSpec(self.request, endpoint.TestEndpoint) self.assertRaises(rest.BadRequest, expectRaiseBadRequest) def test_decode_result_spec_not_a_collection_offset(self): def expectRaiseBadRequest(): offset = 0 - self.make_request('/test') - self.request.args = {'offset': offset} + self.make_request(b'/test') + self.request.args = {b'offset': offset} self.rsrc.decodeResultSpec(self.request, endpoint.TestEndpoint) self.assertRaises(rest.BadRequest, expectRaiseBadRequest) def test_decode_result_spec_not_a_collection_properties(self): expected_props = ['test1', 'test2'] - self.make_request('/test') - self.request.args = {'property': expected_props} + self.make_request(b'/test') + self.request.args = {b'property': expected_props} spec = self.rsrc.decodeResultSpec(self.request, endpoint.TestEndpoint) self.assertEqual(spec.properties[0].values, expected_props) @@ -624,19 +633,19 @@ def test_decode_result_spec_not_a_collection_properties(self): def test_authz_forbidden(self): def deny(request, ep, action, options): - if "test" in ep: + if b"test" in ep: raise authz.Forbidden("no no") return None self.master.www.assertUserAllowed = deny - yield self.render_resource(self.rsrc, '/test') + yield self.render_resource(self.rsrc, b'/test') self.assertRestAuthError(message=re.compile('no no'), responseCode=403) def assertRestAuthError(self, message, responseCode=400): got = {} - got['contentType'] = self.request.headers['content-type'] + got['contentType'] = self.request.headers[b'content-type'] got['responseCode'] = self.request.responseCode - content = json.loads(self.request.written) + content = json.loads(bytes2NativeString(self.request.written)) if 'error' not in content: self.fail("response does not have proper error form: %r" @@ -644,7 +653,7 @@ def assertRestAuthError(self, message, responseCode=400): got['error'] = content['error'] exp = {} - exp['contentType'] = ['text/plain; charset=utf-8'] + exp['contentType'] = [b'text/plain; charset=utf-8'] exp['responseCode'] = responseCode exp['error'] = message @@ -673,9 +682,9 @@ def allow(*args, **kw): def assertJsonRpcError(self, message, responseCode=400, jsonrpccode=None): got = {} - got['contentType'] = self.request.headers['content-type'] + got['contentType'] = self.request.headers[b'content-type'] got['responseCode'] = self.request.responseCode - content = json.loads(self.request.written) + content = json.loads(bytes2NativeString(self.request.written)) if ('error' not in content or sorted(content['error'].keys()) != ['code', 'message']): self.fail("response does not have proper error form: %r" @@ -683,7 +692,7 @@ def assertJsonRpcError(self, message, responseCode=400, jsonrpccode=None): got['error'] = content['error'] exp = {} - exp['contentType'] = ['application/json'] + exp['contentType'] = [b'application/json'] exp['responseCode'] = responseCode exp['error'] = {'code': jsonrpccode, 'message': message} @@ -698,7 +707,7 @@ def assertJsonRpcError(self, message, responseCode=400, jsonrpccode=None): @defer.inlineCallbacks def test_invalid_path(self): - yield self.render_control_resource(self.rsrc, '/not/found') + yield self.render_control_resource(self.rsrc, b'/not/found') self.assertJsonRpcError( message='Invalid path: not/found', jsonrpccode=JSONRPC_CODES['invalid_request'], @@ -706,7 +715,7 @@ def test_invalid_path(self): @defer.inlineCallbacks def test_invalid_action(self): - yield self.render_control_resource(self.rsrc, '/test', action='nosuch') + yield self.render_control_resource(self.rsrc, b'/test', action='nosuch') self.assertJsonRpcError( message='invalid control action', jsonrpccode=JSONRPC_CODES['method_not_found'], @@ -714,7 +723,7 @@ def test_invalid_action(self): @defer.inlineCallbacks def test_invalid_json(self): - yield self.render_control_resource(self.rsrc, '/test', + yield self.render_control_resource(self.rsrc, b'/test', requestJson="{abc") self.assertJsonRpcError( message=re.compile('^JSON parse error'), @@ -722,7 +731,7 @@ def test_invalid_json(self): @defer.inlineCallbacks def test_invalid_content_type(self): - yield self.render_control_resource(self.rsrc, '/test', + yield self.render_control_resource(self.rsrc, b'/test', requestJson='{"jsonrpc": "2.0", "method": "foo",' '"id":"abcdef", "params": {}}', content_type='application/x-www-form-urlencoded') @@ -732,7 +741,7 @@ def test_invalid_content_type(self): @defer.inlineCallbacks def test_list_request(self): - yield self.render_control_resource(self.rsrc, '/test', + yield self.render_control_resource(self.rsrc, b'/test', requestJson="[1,2]") self.assertJsonRpcError( message="JSONRPC batch requests are not supported", @@ -740,7 +749,7 @@ def test_list_request(self): @defer.inlineCallbacks def test_bad_req_type(self): - yield self.render_control_resource(self.rsrc, '/test', + yield self.render_control_resource(self.rsrc, b'/test', requestJson='"a string?!"') self.assertJsonRpcError( message="JSONRPC root object must be an object", @@ -748,7 +757,7 @@ def test_bad_req_type(self): @defer.inlineCallbacks def do_test_invalid_req(self, requestJson, message): - yield self.render_control_resource(self.rsrc, '/test', + yield self.render_control_resource(self.rsrc, b'/test', requestJson=requestJson) self.assertJsonRpcError( message=message, @@ -801,7 +810,7 @@ def test_bad_req_params_type(self): @defer.inlineCallbacks def test_valid(self): - yield self.render_control_resource(self.rsrc, '/test/13', + yield self.render_control_resource(self.rsrc, b'/test/13', action="testy", params={'foo': 3, 'bar': 5}) self.assertRequest( contentJson={ @@ -814,12 +823,12 @@ def test_valid(self): 'kwargs': {'testid': 13}, }, }, - contentType='application/json', + contentType=b'application/json', responseCode=200) @defer.inlineCallbacks def test_valid_int_id(self): - yield self.render_control_resource(self.rsrc, '/test/13', + yield self.render_control_resource(self.rsrc, b'/test/13', action="testy", params={'foo': 3, 'bar': 5}, id=1823) self.assertRequest( contentJson={ @@ -833,12 +842,12 @@ def test_valid_int_id(self): 'kwargs': {'testid': 13}, }, }, - contentType='application/json', + contentType=b'application/json', responseCode=200) @defer.inlineCallbacks def test_valid_fails(self): - yield self.render_control_resource(self.rsrc, '/test/13', + yield self.render_control_resource(self.rsrc, b'/test/13', action="fail") self.assertJsonRpcError( message=re.compile('^RuntimeError'), @@ -850,13 +859,12 @@ def test_valid_fails(self): @defer.inlineCallbacks def test_authz_forbidden(self): - @defer.inlineCallbacks def deny(request, ep, action, options): - if "13" in ep: + if b"13" in ep: raise authz.Forbidden("no no") - defer.returnValue(None) + return None self.master.www.assertUserAllowed = deny - yield self.render_control_resource(self.rsrc, '/test/13', + yield self.render_control_resource(self.rsrc, b'/test/13', action="fail") self.assertJsonRpcError( message=re.compile('no no'), @@ -868,12 +876,12 @@ class ContentTypeParser(unittest.TestCase): def test_simple(self): self.assertEqual( - rest.ContentTypeParser("application/json").gettype(), "application/json") + rest.ContentTypeParser(b"application/json").gettype(), "application/json") def test_complex(self): - self.assertEqual(rest.ContentTypeParser("application/json; Charset=UTF-8").gettype(), + self.assertEqual(rest.ContentTypeParser(b"application/json; Charset=UTF-8").gettype(), "application/json") def test_text(self): self.assertEqual( - rest.ContentTypeParser("text/plain; Charset=UTF-8").gettype(), "text/plain") + rest.ContentTypeParser(b"text/plain; Charset=UTF-8").gettype(), "text/plain") diff --git a/master/buildbot/test/unit/test_www_service.py b/master/buildbot/test/unit/test_www_service.py index f0f7f41b88f..ee76fb1162b 100644 --- a/master/buildbot/test/unit/test_www_service.py +++ b/master/buildbot/test/unit/test_www_service.py @@ -149,7 +149,7 @@ def test_setupSite(self): # root root = site.resource req = mock.Mock() - self.assertIsInstance(root.getChildWithDefault('api', req), + self.assertIsInstance(root.getChildWithDefault(b'api', req), rest.RestRootResource) def test_setupSiteWithProtectedHook(self): @@ -165,7 +165,7 @@ def test_setupSiteWithProtectedHook(self): # root root = site.resource req = mock.Mock() - self.assertIsInstance(root.getChildWithDefault('change_hook', req), + self.assertIsInstance(root.getChildWithDefault(b'change_hook', req), HTTPAuthSessionWrapper) @defer.inlineCallbacks @@ -179,7 +179,7 @@ def test_setupSiteWithHook(self): # root root = site.resource req = mock.Mock() - ep = root.getChildWithDefault('change_hook', req) + ep = root.getChildWithDefault(b'change_hook', req) self.assertIsInstance(ep, change_hook.ChangeHookResource) @@ -191,8 +191,8 @@ def test_setupSiteWithHook(self): # now configured self.assertEqual(ep.dialects, {'base': True}) - rsrc = self.svc.site.resource.getChildWithDefault('change_hook', mock.Mock()) - path = '/change_hook/base' + rsrc = self.svc.site.resource.getChildWithDefault(b'change_hook', mock.Mock()) + path = b'/change_hook/base' request = test_www_hooks_base._prepare_request({}) self.master.addChange = mock.Mock() yield self.render_resource(rsrc, path, request=request) @@ -211,14 +211,16 @@ def test_setupSiteWithHookAndAuth(self): self.svc.setupSite(new_config) yield self.svc.reconfigServiceWithBuildbotConfig(new_config) - rsrc = self.svc.site.resource.getChildWithDefault('', mock.Mock()) + rsrc = self.svc.site.resource.getChildWithDefault(b'', mock.Mock()) - res = yield self.render_resource(rsrc, '') - self.assertIn('{"type": "file"}', res) + res = yield self.render_resource(rsrc, b'') + self.assertIn(b'{"type": "file"}', res) - rsrc = self.svc.site.resource.getChildWithDefault('change_hook', mock.Mock()) - res = yield self.render_resource(rsrc, '/change_hook/base') - # as UnauthorizedResource is in private namespace, we cannot use assertIsInstance :-( + rsrc = self.svc.site.resource.getChildWithDefault( + b'change_hook', mock.Mock()) + res = yield self.render_resource(rsrc, b'/change_hook/base') + # as UnauthorizedResource is in private namespace, we cannot use + # assertIsInstance :-( self.assertIn('UnauthorizedResource', repr(res)) @@ -269,10 +271,11 @@ class FakeChannel(object): def isSecure(self): return False request = Request(FakeChannel(), False) - request.sitepath = ["bb"] + request.sitepath = [b"bb"] session.updateSession(request) self.assertEqual(len(request.cookies), 1) - name, value = request.cookies[0].split(";")[0].split("=") - decoded = jwt.decode(value, self.SECRET, algorithm=service.SESSION_SECRET_ALGORITHM) + name, value = request.cookies[0].split(b";")[0].split(b"=") + decoded = jwt.decode(value, self.SECRET, + algorithm=service.SESSION_SECRET_ALGORITHM) self.assertEqual(decoded['user_info'], {'anonymous': True}) self.assertIn('exp', decoded) diff --git a/master/buildbot/test/unit/test_www_sse.py b/master/buildbot/test/unit/test_www_sse.py index 810f7994768..01982bd8047 100644 --- a/master/buildbot/test/unit/test_www_sse.py +++ b/master/buildbot/test/unit/test_www_sse.py @@ -23,34 +23,37 @@ from buildbot.test.unit import test_data_changes from buildbot.test.util import www +from buildbot.util import bytes2NativeString from buildbot.util import datetime2epoch +from buildbot.util import unicode2bytes from buildbot.www import sse class EventResource(www.WwwTestMixin, unittest.TestCase): def setUp(self): - self.master = master = self.make_master(url='h:/a/b/') + self.master = master = self.make_master(url=b'h:/a/b/') self.sse = sse.EventResource(master) def test_simpleapi(self): - self.render_resource(self.sse, '/changes/*/*') + self.render_resource(self.sse, b'/changes/*/*') self.readUUID(self.request) self.assertReceivesChangeNewMessage(self.request) self.assertEqual(self.request.finished, False) def test_listen(self): - self.render_resource(self.sse, '/listen/changes/*/*') + self.render_resource(self.sse, b'/listen/changes/*/*') self.readUUID(self.request) self.assertReceivesChangeNewMessage(self.request) self.assertEqual(self.request.finished, False) def test_listen_add_then_close(self): - self.render_resource(self.sse, '/listen') + self.render_resource(self.sse, b'/listen') request = self.request self.request = None uuid = self.readUUID(request) - self.render_resource(self.sse, '/add/' + uuid + "/changes/*/*") + self.render_resource(self.sse, b'/add/' + + unicode2bytes(uuid) + b"/changes/*/*") self.assertReceivesChangeNewMessage(request) self.assertEqual(self.request.finished, True) self.assertEqual(request.finished, False) @@ -59,60 +62,63 @@ def test_listen_add_then_close(self): AssertionError, self.assertReceivesChangeNewMessage, request) def test_listen_add_then_remove(self): - self.render_resource(self.sse, '/listen') + self.render_resource(self.sse, b'/listen') request = self.request uuid = self.readUUID(request) - self.render_resource(self.sse, '/add/' + uuid + "/changes/*/*") + self.render_resource(self.sse, b'/add/' + + unicode2bytes(uuid) + b"/changes/*/*") self.assertReceivesChangeNewMessage(request) self.assertEqual(request.finished, False) - self.render_resource(self.sse, '/remove/' + uuid + "/changes/*/*") + self.render_resource(self.sse, b'/remove/' + + unicode2bytes(uuid) + b"/changes/*/*") self.assertRaises( AssertionError, self.assertReceivesChangeNewMessage, request) def test_listen_add_nouuid(self): - self.render_resource(self.sse, '/listen') + self.render_resource(self.sse, b'/listen') request = self.request self.readUUID(request) - self.render_resource(self.sse, '/add/') + self.render_resource(self.sse, b'/add/') self.assertEqual(self.request.finished, True) self.assertEqual(self.request.responseCode, 400) - self.assertIn("need uuid", self.request.written) + self.assertIn(b"need uuid", self.request.written) def test_listen_add_baduuid(self): - self.render_resource(self.sse, '/listen') + self.render_resource(self.sse, b'/listen') request = self.request self.readUUID(request) - self.render_resource(self.sse, '/add/foo') + self.render_resource(self.sse, b'/add/foo') self.assertEqual(self.request.finished, True) self.assertEqual(self.request.responseCode, 400) - self.assertIn("unknown uuid", self.request.written) + self.assertIn(b"unknown uuid", self.request.written) def readEvent(self, request): kw = {} hasEmptyLine = False for line in request.written.splitlines(): - if line.find(":") > 0: - k, v = line.split(": ", 1) - self.assertTrue(k not in kw, k + " in " + str(kw)) + if line.find(b":") > 0: + k, v = line.split(b": ", 1) + self.assertTrue(k not in kw, k + b" in " + + unicode2bytes(str(kw))) kw[k] = v else: - self.assertEqual(line, "") + self.assertEqual(line, b"") hasEmptyLine = True - request.written = "" + request.written = b"" self.assertTrue(hasEmptyLine) return kw def readUUID(self, request): kw = self.readEvent(request) - self.assertEqual(kw["event"], "handshake") - return kw["data"] + self.assertEqual(kw[b"event"], b"handshake") + return kw[b"data"] def assertReceivesChangeNewMessage(self, request): self.master.mq.callConsumer( - ("changes", "500", "new"), test_data_changes.Change.changeEvent) + (b"changes", b"500", b"new"), test_data_changes.Change.changeEvent) kw = self.readEvent(request) - self.assertEqual(kw["event"], "event") - msg = json.loads(kw["data"]) + self.assertEqual(kw[b"event"], b"event") + msg = json.loads(bytes2NativeString(kw[b"data"])) self.assertEqual(msg["key"], [u'changes', u'500', u'new']) self.assertEqual(msg["message"], json.loads( json.dumps(test_data_changes.Change.changeEvent, default=self._toJson))) diff --git a/master/buildbot/test/util/validation.py b/master/buildbot/test/util/validation.py index dcfa34d547c..3f4b654d9c3 100644 --- a/master/buildbot/test/util/validation.py +++ b/master/buildbot/test/util/validation.py @@ -25,6 +25,7 @@ import re from buildbot.util import UTC +from buildbot.util import bytes2NativeString # Base class @@ -376,7 +377,7 @@ def validate(self, name, arg_object): message['builders'] = Selector() message['builders'].add(None, MessageValidator( - events=['started', 'stopped'], + events=[b'started', b'stopped'], messageValidator=DictValidator( builderid=IntValidator(), masterid=IntValidator(), @@ -419,10 +420,10 @@ def validate(self, name, arg_object): parent_buildid=NoneOk(IntValidator()), parent_relationship=NoneOk(StringValidator()), ) -_buildsetEvents = ['new', 'complete'] +_buildsetEvents = [b'new', b'complete'] message['buildsets'] = Selector() -message['buildsets'].add(lambda k: k[-1] == 'new', +message['buildsets'].add(lambda k: k[-1] == b'new', MessageValidator( events=_buildsetEvents, messageValidator=DictValidator( @@ -462,7 +463,7 @@ def validate(self, name, arg_object): message['buildrequests'] = Selector() message['buildrequests'].add(None, MessageValidator( - events=['new', 'claimed', 'unclaimed'], + events=[b'new', b'claimed', b'unclaimed'], messageValidator=DictValidator( # TODO: probably wrong! brid=IntValidator(), @@ -476,7 +477,7 @@ def validate(self, name, arg_object): message['changes'] = Selector() message['changes'].add(None, MessageValidator( - events=['new'], + events=[b'new'], messageValidator=DictValidator( changeid=IntValidator(), parent_changeids=ListValidator(IntValidator()), @@ -547,7 +548,7 @@ def validate(self, name, arg_object): state_string=StringValidator(), results=NoneOk(IntValidator()), ) -_buildEvents = ['new', 'complete'] +_buildEvents = [b'new', b'complete'] message['builds'] = Selector() message['builds'].add(None, @@ -592,7 +593,7 @@ def validate(self, name, arg_object): urls=ListValidator(StringValidator()), hidden=BooleanValidator(), ) -_stepEvents = ['new', 'complete'] +_stepEvents = [b'new', b'complete'] message['steps'] = Selector() message['steps'].add(None, @@ -656,7 +657,7 @@ def verifyMessage(testcase, routingKey, message_): # the "type" of the message is identified by last path name # -1 being the event, and -2 the id. - validator = message[routingKey[-3]] + validator = message[bytes2NativeString(routingKey[-3])] _verify(testcase, validator, '', (routingKey, (routingKey, message_))) diff --git a/master/buildbot/test/util/www.py b/master/buildbot/test/util/www.py index 8de7dabf0b3..46640d8df43 100644 --- a/master/buildbot/test/util/www.py +++ b/master/buildbot/test/util/www.py @@ -32,6 +32,8 @@ from twisted.web import server from buildbot.test.fake import fakemaster +from buildbot.util import bytes2NativeString +from buildbot.util import unicode2bytes from buildbot.www import auth from buildbot.www import authz @@ -46,20 +48,20 @@ def updateSession(self, request): class FakeRequest(object): - written = '' + written = b'' finished = False redirected_to = None rendered_resource = None failure = None - method = 'GET' - path = '/req.path' + method = b'GET' + path = b'/req.path' responseCode = 200 def __init__(self, path=None): self.headers = {} self.input_headers = {} self.prepath = [] - x = path.split('?', 1) + x = path.split(b'?', 1) if len(x) == 1: self.path = path self.args = {} @@ -68,7 +70,10 @@ def __init__(self, path=None): self.path = path self.args = parse_qs(argstring, 1) self.uri = self.path - self.postpath = list(map(urlunquote, path[1:].split('/'))) + self.postpath = [] + for p in path[1:].split(b'/'): + path = urlunquote(bytes2NativeString(p)) + self.postpath.append(unicode2bytes(path)) self.deferred = defer.Deferred() @@ -147,23 +152,23 @@ def make_master(self, url=None, **kwargs): self.master.authz.setMaster(self.master) return master - def make_request(self, path=None, method='GET'): + def make_request(self, path=None, method=b'GET'): self.request = FakeRequest(path) self.request.session = self.master.session self.request.method = method return self.request - def render_resource(self, rsrc, path='/', accept=None, method='GET', + def render_resource(self, rsrc, path=b'/', accept=None, method=b'GET', origin=None, access_control_request_method=None, extraHeaders=None, request=None): if not request: request = self.make_request(path, method=method) if accept: - request.input_headers['accept'] = accept + request.input_headers[b'accept'] = accept if origin: - request.input_headers['origin'] = origin + request.input_headers[b'origin'] = origin if access_control_request_method: - request.input_headers['access-control-request-method'] = \ + request.input_headers[b'access-control-request-method'] = \ access_control_request_method if extraHeaders is not None: request.input_headers.update(extraHeaders) @@ -175,16 +180,16 @@ def render_resource(self, rsrc, path='/', accept=None, method='GET', request.finish() return request.deferred - def render_control_resource(self, rsrc, path='/', params={}, + def render_control_resource(self, rsrc, path=b'/', params={}, requestJson=None, action="notfound", id=None, - content_type='application/json'): + content_type=b'application/json'): # pass *either* a request or postpath id = id or self.UUID request = self.make_request(path) - request.method = "POST" + request.method = b"POST" request.content = NativeStringIO(requestJson or json.dumps( {"jsonrpc": "2.0", "method": action, "params": params, "id": id})) - request.input_headers = {'content-type': content_type} + request.input_headers = {b'content-type': content_type} rv = rsrc.render(request) if rv != server.NOT_DONE_YET: d = defer.succeed(rv) @@ -193,7 +198,7 @@ def render_control_resource(self, rsrc, path='/', params={}, @d.addCallback def check(_json): - res = json.loads(_json) + res = json.loads(bytes2NativeString(_json)) self.assertIn("jsonrpc", res) self.assertEqual(res["jsonrpc"], "2.0") if not requestJson: @@ -209,10 +214,10 @@ def assertRequest(self, content=None, contentJson=None, contentType=None, got['content'] = self.request.written exp['content'] = content if contentJson is not None: - got['contentJson'] = json.loads(self.request.written) + got['contentJson'] = json.loads(bytes2NativeString(self.request.written)) exp['contentJson'] = contentJson if contentType is not None: - got['contentType'] = self.request.headers['content-type'] + got['contentType'] = self.request.headers[b'content-type'] exp['contentType'] = [contentType] if responseCode is not None: got['responseCode'] = str(self.request.responseCode) diff --git a/master/buildbot/util/__init__.py b/master/buildbot/util/__init__.py index 5a2388e51b2..a8b4d9715f4 100644 --- a/master/buildbot/util/__init__.py +++ b/master/buildbot/util/__init__.py @@ -398,14 +398,14 @@ def do_stop(r): def string2boolean(str): return { - 'on': True, - 'true': True, - 'yes': True, - '1': True, - 'off': False, - 'false': False, - 'no': False, - '0': False, + b'on': True, + b'true': True, + b'yes': True, + b'1': True, + b'off': False, + b'false': False, + b'no': False, + b'0': False, }[str.lower()] diff --git a/master/buildbot/www/auth.py b/master/buildbot/www/auth.py index 801cc4f4ef3..be2cd5902fc 100644 --- a/master/buildbot/www/auth.py +++ b/master/buildbot/www/auth.py @@ -38,9 +38,9 @@ class AuthRootResource(resource.Resource): def getChild(self, path, request): # return dynamically generated resources - if path == 'login': + if path == b'login': return self.master.www.auth.getLoginResource() - elif path == 'logout': + elif path == b'logout': return self.master.www.auth.getLogoutResource() return resource.Resource.getChild(self, path, request) @@ -196,4 +196,4 @@ def render_GET(self, request): session = request.getSession() session.expire() session.updateSession(request) - return "" + return b'' diff --git a/master/buildbot/www/avatar.py b/master/buildbot/www/avatar.py index 76c4942f21c..ac35e467154 100644 --- a/master/buildbot/www/avatar.py +++ b/master/buildbot/www/avatar.py @@ -57,12 +57,12 @@ def getUserAvatar(self, email, size, defaultAvatarUrl): class AvatarResource(resource.Resource): # enable reconfigResource calls needsReconfig = True - defaultAvatarUrl = "img/nobody.png" + defaultAvatarUrl = b"img/nobody.png" def reconfigResource(self, new_config): self.avatarMethods = new_config.www.get('avatar_methods', []) self.defaultAvatarFullUrl = urljoin( - new_config.buildbotURL, self.defaultAvatarUrl) + unicode2bytes(new_config.buildbotURL), unicode2bytes(self.defaultAvatarUrl)) self.cache = {} # ensure the avatarMethods is a iterable if isinstance(self.avatarMethods, AvatarBase): @@ -73,8 +73,8 @@ def render_GET(self, request): @defer.inlineCallbacks def renderAvatar(self, request): - email = request.args.get("email", [""])[0] - size = request.args.get("size", 32) + email = request.args.get(b"email", [b""])[0] + size = request.args.get(b"size", 32) if self.cache.get(email): r = self.cache[email] for method in self.avatarMethods: @@ -84,8 +84,8 @@ def renderAvatar(self, request): self.cache[email] = r raise if res is not None: - request.setHeader('content-type', res[0]) - request.setHeader('content-length', len(res[1])) + request.setHeader(b'content-type', res[0]) + request.setHeader(b'content-length', len(res[1])) request.write(res[1]) return raise resource.Redirect(self.defaultAvatarUrl) diff --git a/master/buildbot/www/oauth2.py b/master/buildbot/www/oauth2.py index 04e7ab63002..2471d192178 100644 --- a/master/buildbot/www/oauth2.py +++ b/master/buildbot/www/oauth2.py @@ -45,8 +45,8 @@ def render_POST(self, request): @defer.inlineCallbacks def renderLogin(self, request): - code = request.args.get("code", [""])[0] - token = request.args.get("token", [""])[0] + code = request.args.get(b"code", [""])[0] + token = request.args.get(b"token", [""])[0] if not token and not code: url = request.args.get("redirect", [None])[0] url = yield self.auth.getLoginURL(url) diff --git a/master/buildbot/www/resource.py b/master/buildbot/www/resource.py index e157039cd94..b1ed98ec954 100644 --- a/master/buildbot/www/resource.py +++ b/master/buildbot/www/resource.py @@ -22,6 +22,8 @@ from twisted.web import server from twisted.web.error import Error +from buildbot.util import unicode2bytes + class Redirect(Error): @@ -56,7 +58,7 @@ def reconfigResource(self, new_config): def asyncRenderHelper(self, request, _callable, writeError=None): def writeErrorDefault(msg, errcode=400): request.setResponseCode(errcode) - request.setHeader('content-type', 'text/plain; charset=utf-8') + request.setHeader(b'content-type', b'text/plain; charset=utf-8') request.write(msg) request.finish() if writeError is None: @@ -88,13 +90,14 @@ def failHttpRedirect(f): def failHttpError(f): f.trap(Error) e = f.value - writeError(e.message, errcode=int(e.status)) + message = unicode2bytes(e.message) + writeError(message, errcode=int(e.status)) @d.addErrback def fail(f): log.err(f, 'While rendering resource:') try: - writeError('internal error - see logs', errcode=500) + writeError(b'internal error - see logs', errcode=500) except Exception: try: request.finish() diff --git a/master/buildbot/www/rest.py b/master/buildbot/www/rest.py index c45528797d4..85754c6d3c0 100644 --- a/master/buildbot/www/rest.py +++ b/master/buildbot/www/rest.py @@ -32,7 +32,10 @@ from buildbot.data import exceptions from buildbot.data import resultspec +from buildbot.util import bytes2NativeString +from buildbot.util import bytes2unicode from buildbot.util import toJson +from buildbot.util import unicode2bytes from buildbot.www import resource from buildbot.www.authz import Forbidden @@ -54,12 +57,13 @@ def __init__(self, contenttype): self.typeheader = contenttype def gettype(self): - mimetype, options = cgi.parse_header(self.typeheader) + mimetype, options = cgi.parse_header( + bytes2NativeString(self.typeheader)) return mimetype -URL_ENCODED = "application/x-www-form-urlencoded" -JSON_ENCODED = "application/json" +URL_ENCODED = b"application/x-www-form-urlencoded" +JSON_ENCODED = b"application/json" class RestRootResource(resource.Resource): @@ -79,17 +83,20 @@ def __init__(self, master): if version < min_vers: continue child = klass(master) - self.putChild('v%d' % (version,), child) + child_path = 'v{}'.format(version) + child_path = unicode2bytes(child_path) + self.putChild(child_path, child) if version == latest: - self.putChild('latest', child) + self.putChild(b'latest', child) def render(self, request): - request.setHeader("content-type", JSON_ENCODED) + request.setHeader(b"content-type", JSON_ENCODED) min_vers = self.master.config.www.get('rest_minimum_version', 0) api_versions = dict(('v%d' % v, '%sapi/v%d' % (self.base_url, v)) for v in self.version_classes if v > min_vers) - return json.dumps(dict(api_versions=api_versions)) + data = json.dumps(dict(api_versions=api_versions)) + return unicode2bytes(data) JSONRPC_CODES = dict(parse_error=-32700, @@ -121,35 +128,42 @@ class V2RootResource(resource.Resource): def getEndpoint(self, request): # note that trailing slashes are not allowed - return self.master.data.getEndpoint(tuple(request.postpath)) + request_postpath = [bytes2NativeString(p) for p in request.postpath] + return self.master.data.getEndpoint(tuple(request_postpath)) @contextmanager def handleErrors(self, writeError): try: yield except exceptions.InvalidPathError as e: - writeError(e.args[0] or "invalid path", errcode=404, + msg = unicode2bytes(e.args[0]) + writeError(msg or b"invalid path", errcode=404, jsonrpccode=JSONRPC_CODES['invalid_request']) return except exceptions.InvalidControlException as e: - writeError(str(e) or "invalid control action", errcode=501, + msg = unicode2bytes(str(e)) + writeError(msg or b"invalid control action", errcode=501, jsonrpccode=JSONRPC_CODES["method_not_found"]) return except BadRequest as e: - writeError(e.args[0] or "invalid request", errcode=400, + msg = unicode2bytes(e.args[0]) + writeError(msg or b"invalid request", errcode=400, jsonrpccode=JSONRPC_CODES["method_not_found"]) return except BadJsonRpc2 as e: - writeError(e.message, errcode=400, jsonrpccode=e.jsonrpccode) + msg = unicode2bytes(e.message) + writeError(msg, errcode=400, jsonrpccode=e.jsonrpccode) return except Forbidden as e: # There is nothing in jsonrc spec about forbidden error, so pick # invalid request + msg = unicode2bytes(e.message) writeError( - e.message, errcode=403, jsonrpccode=JSONRPC_CODES["invalid_request"]) + msg, errcode=403, jsonrpccode=JSONRPC_CODES["invalid_request"]) return except Exception as e: log.err(_why='while handling API request') + msg = unicode2bytes(repr(e)) writeError(repr(e), errcode=500, jsonrpccode=JSONRPC_CODES["internal_error"]) return @@ -160,7 +174,7 @@ def decodeJsonRPC2(self, request): # Verify the content-type. Browsers are easily convinced to send # POST data to arbitrary URLs via 'form' elements, but they won't # use the application/json content-type. - if ContentTypeParser(request.getHeader('content-type')).gettype() != JSON_ENCODED: + if ContentTypeParser(request.getHeader(b'content-type')).gettype() != "application/json": raise BadJsonRpc2('Invalid content-type (use application/json)', JSONRPC_CODES["invalid_request"]) @@ -200,13 +214,16 @@ def renderJsonRpc(self, request): def writeError(msg, errcode=399, jsonrpccode=JSONRPC_CODES["internal_error"]): + msg = bytes2NativeString(msg) if self.debug: log.msg("JSONRPC error: %s" % (msg,)) request.setResponseCode(errcode) - request.setHeader('content-type', JSON_ENCODED) + request.setHeader(b'content-type', JSON_ENCODED) if "error" not in jsonRpcReply: # already filled in by caller jsonRpcReply['error'] = dict(code=jsonrpccode, message=msg) - request.write(json.dumps(jsonRpcReply)) + data = json.dumps(jsonRpcReply) + data = unicode2bytes(data) + request.write(data) with self.handleErrors(writeError): method, id, params = self.decodeJsonRPC2(request) @@ -227,11 +244,12 @@ def writeError(msg, errcode=399, data = json.dumps(jsonRpcReply, default=toJson, sort_keys=True, separators=(',', ':')) - request.setHeader('content-type', JSON_ENCODED) - if request.method == "HEAD": - request.setHeader("content-length", len(data)) - request.write('') + request.setHeader(b'content-type', JSON_ENCODED) + if request.method == b"HEAD": + request.setHeader(b"content-length", len(data)) + request.write(b'') else: + data = unicode2bytes(data) request.write(data) # JSONAPI support @@ -239,54 +257,63 @@ def decodeResultSpec(self, request, endpoint): reqArgs = request.args def checkFields(fields, negOk=False): - for k in fields: + for field in fields: + k = bytes2NativeString(field) if k[0] == '-' and negOk: k = k[1:] if k not in entityType.fieldNames: - raise BadRequest("no such field %r" % (k,)) + raise BadRequest("no such field '{}'".format(k)) entityType = endpoint.rtype.entityType limit = offset = order = fields = None filters, properties = [], [] for arg in reqArgs: - if arg == 'order': - order = reqArgs[arg] + argStr = bytes2NativeString(arg) + if arg == b'order': + order = tuple([bytes2NativeString(o) for o in reqArgs[arg]]) checkFields(order, True) continue - elif arg == 'field': + elif arg == b'field': fields = reqArgs[arg] checkFields(fields, False) continue - elif arg == 'limit': + elif arg == b'limit': try: limit = int(reqArgs[arg][0]) except Exception: raise BadRequest('invalid limit') continue - elif arg == 'offset': + elif arg == b'offset': try: offset = int(reqArgs[arg][0]) except Exception: raise BadRequest('invalid offset') continue - elif arg == 'property': + elif arg == b'property': try: - props = [v.decode('utf-8') for v in reqArgs[arg]] + props = [] + for v in reqArgs[arg]: + if not isinstance(v, (bytes, text_type)): + raise TypeError( + "Invalid type {} for {}".format(type(v), v)) + props.append(bytes2unicode(v)) except Exception: - raise BadRequest('invalid property value for %s' % arg) + raise BadRequest( + 'invalid property value for {}'.format(arg)) properties.append(resultspec.Property(arg, 'eq', props)) continue - elif arg in entityType.fieldNames: - field = entityType.fields[arg] + elif argStr in entityType.fieldNames: + field = entityType.fields[argStr] try: values = [field.valueFromString(v) for v in reqArgs[arg]] except Exception: - raise BadRequest('invalid filter value for %s' % arg) + raise BadRequest( + 'invalid filter value for {}'.format(argStr)) - filters.append(resultspec.Filter(arg, 'eq', values)) + filters.append(resultspec.Filter(argStr, 'eq', values)) continue - elif '__' in arg: - field, op = arg.rsplit('__', 1) + elif '__' in argStr: + field, op = argStr.rsplit('__', 1) args = reqArgs[arg] operators = (resultspec.Filter.singular_operators if len(args) == 1 @@ -297,13 +324,16 @@ def checkFields(fields, negOk=False): values = [fieldType.valueFromString(v) for v in reqArgs[arg]] except Exception: - raise BadRequest('invalid filter value for %s' % arg) + raise BadRequest( + 'invalid filter value for {}'.format(argStr)) filters.append(resultspec.Filter(field, op, values)) continue - raise BadRequest("unrecognized query parameter '%s'" % (arg,)) + raise BadRequest( + "unrecognized query parameter '{}'".format(argStr)) # if ordering or filtering is on a field that's not in fields, bail out if fields: + fields = [bytes2NativeString(f) for f in fields] fieldsSet = set(fields) if order and set([o.lstrip('-') for o in order]) - fieldsSet: raise BadRequest("cannot order on un-selected fields") @@ -323,10 +353,10 @@ def checkFields(fields, negOk=False): return rspec def encodeRaw(self, data, request): - request.setHeader("content-type", - data['mime-type'].encode() + '; charset=utf-8') - request.setHeader("content-disposition", - 'attachment; filename=' + data['filename'].encode()) + request.setHeader(b"content-type", + data['mime-type'].encode() + b'; charset=utf-8') + request.setHeader(b"content-disposition", + b'attachment; filename=' + data['filename'].encode()) request.write(data['raw'].encode('utf-8')) return @@ -336,8 +366,11 @@ def writeError(msg, errcode=404, jsonrpccode=None): if self.debug: log.msg("REST error: %s" % (msg,)) request.setResponseCode(errcode) - request.setHeader('content-type', 'text/plain; charset=utf-8') - request.write(json.dumps(dict(error=msg))) + request.setHeader(b'content-type', b'text/plain; charset=utf-8') + msg = bytes2NativeString(msg) + data = json.dumps(dict(error=msg)) + data = unicode2bytes(data) + request.write(data) with self.handleErrors(writeError): yield self.master.www.assertUserAllowed(request, tuple(request.postpath), request.method, {}) @@ -347,9 +380,11 @@ def writeError(msg, errcode=404, jsonrpccode=None): rspec = self.decodeResultSpec(request, ep) data = yield ep.get(rspec, kwargs) if data is None: - writeError(("not found while getting from %s with " - "arguments %s and %s") % (repr(ep), repr(rspec), - str(kwargs)), errcode=404) + msg = ("not found while getting from {} with " + "arguments {} and {}").format(repr(ep), repr(rspec), + str(kwargs)) + msg = unicode2bytes(msg) + writeError(msg, errcode=404) return if ep.isRaw: @@ -385,22 +420,23 @@ def writeError(msg, errcode=404, jsonrpccode=None): # accepts text/html or text/plain, the JSON will be rendered in a # readable, multiline format. - if 'application/json' in (request.getHeader('accept') or ''): + if b'application/json' in (request.getHeader(b'accept') or b''): compact = True - request.setHeader("content-type", - 'application/json; charset=utf-8') + request.setHeader(b"content-type", + b'application/json; charset=utf-8') else: compact = False - request.setHeader("content-type", - 'text/plain; charset=utf-8') + request.setHeader(b"content-type", + b'text/plain; charset=utf-8') # set up caching if self.cache_seconds: now = datetime.datetime.utcnow() expires = now + datetime.timedelta(seconds=self.cache_seconds) - request.setHeader("Expires", - expires.strftime("%a, %d %b %Y %H:%M:%S GMT")) - request.setHeader("Pragma", "no-cache") + expiresBytes = unicode2bytes( + expires.strftime("%a, %d %b %Y %H:%M:%S GMT")) + request.setHeader(b"Expires", expiresBytes) + request.setHeader(b"Pragma", b"no-cache") # filter out blanks if necessary and render the data if compact: @@ -410,20 +446,22 @@ def writeError(msg, errcode=404, jsonrpccode=None): data = json.dumps(data, default=toJson, sort_keys=True, indent=2) - if request.method == "HEAD": - request.setHeader("content-length", len(data)) + if request.method == b"HEAD": + request.setHeader(b"content-length", len(data)) else: + data = unicode2bytes(data) request.write(data) def reconfigResource(self, new_config): # buildbotURL may contain reverse proxy path, Origin header is just # scheme + host + port - buildbotURL = urlparse(new_config.buildbotURL) - origin_self = buildbotURL.scheme + "://" + buildbotURL.netloc + buildbotURL = urlparse(unicode2bytes(new_config.buildbotURL)) + origin_self = buildbotURL.scheme + b"://" + buildbotURL.netloc # pre-translate the origin entries in the config - self.origins = [re.compile(fnmatch.translate(o.lower())) - for o in new_config.www.get('allowed_origins', - [origin_self])] + self.origins = [] + for o in new_config.www.get('allowed_origins', [origin_self]): + origin = bytes2NativeString(o).lower() + self.origins.append(re.compile(fnmatch.translate(origin))) # and copy some other flags self.debug = new_config.www.get('debug') @@ -431,15 +469,20 @@ def reconfigResource(self, new_config): def render(self, request): def writeError(msg, errcode=400): + msg = bytes2NativeString(msg) if self.debug: log.msg("HTTP error: %s" % (msg,)) request.setResponseCode(errcode) - request.setHeader('content-type', 'text/plain; charset=utf-8') - if request.method == 'POST': + request.setHeader(b'content-type', b'text/plain; charset=utf-8') + if request.method == b'POST': # jsonRPC callers want the error message in error.message - request.write(json.dumps(dict(error=dict(message=msg)))) + data = json.dumps(dict(error=dict(message=msg))) + data = unicode2bytes(data) + request.write(data) else: - request.write(json.dumps(dict(error=msg))) + data = json.dumps(dict(error=msg)) + data = unicode2bytes(data) + request.write(data) request.finish() return self.asyncRenderHelper(request, self.asyncRender, writeError) @@ -450,17 +493,17 @@ def asyncRender(self, request): origins = self.origins if origins is not None: isPreflight = False - reqOrigin = request.getHeader('origin') + reqOrigin = request.getHeader(b'origin') if reqOrigin: err = None reqOrigin = reqOrigin.lower() - if not any(o.match(reqOrigin) for o in self.origins): + if not any(o.match(bytes2NativeString(reqOrigin)) for o in self.origins): err = "invalid origin" - elif request.method == 'OPTIONS': + elif request.method == b'OPTIONS': preflightMethod = request.getHeader( - 'access-control-request-method') - if preflightMethod not in ('GET', 'POST', 'HEAD'): - err = 'invalid method' + b'access-control-request-method') + if preflightMethod not in (b'GET', b'POST', b'HEAD'): + err = b'invalid method' isPreflight = True if err: raise Error(400, err) @@ -469,22 +512,22 @@ def asyncRender(self, request): # Content-Type header is included here because CORS considers # content types other than form data and text/plain to not be # simple. - request.setHeader("access-control-allow-origin", reqOrigin) - request.setHeader("access-control-allow-headers", - "Content-Type") - request.setHeader("access-control-max-age", '3600') + request.setHeader(b"access-control-allow-origin", reqOrigin) + request.setHeader(b"access-control-allow-headers", + b"Content-Type") + request.setHeader(b"access-control-max-age", b'3600') # if this was a preflight request, we're done if isPreflight: - defer.returnValue("") + defer.returnValue(b"") # based on the method, this is either JSONRPC or REST - if request.method == 'POST': + if request.method == b'POST': res = yield self.renderJsonRpc(request) - elif request.method in ('GET', 'HEAD'): + elif request.method in (b'GET', b'HEAD'): res = yield self.renderRest(request) else: - raise Error(400, "invalid HTTP method") + raise Error(400, b"invalid HTTP method") defer.returnValue(res) diff --git a/master/buildbot/www/service.py b/master/buildbot/www/service.py index 9d49f24a7c8..c57e72fdb61 100644 --- a/master/buildbot/www/service.py +++ b/master/buildbot/www/service.py @@ -38,6 +38,7 @@ from zope.interface import implementer from buildbot.plugins.db import get_plugins +from buildbot.util import bytes2NativeString from buildbot.util import service from buildbot.www import config as wwwconfig from buildbot.www import auth @@ -145,6 +146,7 @@ def __init__(self, root, logPath, rotateLength, maxRotatedFiles): self.session_secret = None def _openLogFile(self, path): + self._nativeize = True return LogFile.fromFullPath( path, rotateLength=self.rotateLength, maxRotatedFiles=self.maxRotatedFiles) @@ -277,23 +279,23 @@ def setupSite(self, new_config): "configured" % (plugin_name,)) # / - root.putChild('', wwwconfig.IndexResource( + root.putChild(b'', wwwconfig.IndexResource( self.master, self.apps.get('base').static_dir)) # /auth - root.putChild('auth', auth.AuthRootResource(self.master)) + root.putChild(b'auth', auth.AuthRootResource(self.master)) # /avatar - root.putChild('avatar', avatar.AvatarResource(self.master)) + root.putChild(b'avatar', avatar.AvatarResource(self.master)) # /api - root.putChild('api', rest.RestRootResource(self.master)) + root.putChild(b'api', rest.RestRootResource(self.master)) # /ws - root.putChild('ws', ws.WsResource(self.master)) + root.putChild(b'ws', ws.WsResource(self.master)) # /sse - root.putChild('sse', sse.EventResource(self.master)) + root.putChild(b'sse', sse.EventResource(self.master)) # /change_hook resource_obj = change_hook.ChangeHookResource(master=self.master) @@ -303,7 +305,7 @@ def setupSite(self, new_config): if change_hook_auth is not None: resource_obj = self.setupProtectedResource( resource_obj, change_hook_auth) - root.putChild("change_hook", resource_obj) + root.putChild(b"change_hook", resource_obj) self.root = root @@ -351,7 +353,7 @@ def create_session_secret(): # and other runs of this master # we encode that in hex for db storage convenience - return hexlify(os.urandom(int(SESSION_SECRET_LENGTH / 8))) + return bytes2NativeString(hexlify(os.urandom(int(SESSION_SECRET_LENGTH / 8)))) session_secret = yield state.atomicCreateState(objectid, "session_secret", create_session_secret) self.site.setSessionSecret(session_secret) diff --git a/master/buildbot/www/sse.py b/master/buildbot/www/sse.py index daad2a3af16..e0145ed364b 100644 --- a/master/buildbot/www/sse.py +++ b/master/buildbot/www/sse.py @@ -25,7 +25,9 @@ from twisted.web import server from buildbot.data.exceptions import InvalidPathError +from buildbot.util import bytes2NativeString from buildbot.util import toJson +from buildbot.util import unicode2bytes class Consumer(object): @@ -44,10 +46,12 @@ def stopConsuming(self, key=None): def onMessage(self, event, data): request = self.request - msg = dict(key=event, message=data) - request.write("event: " + "event" + "\n") - request.write("data: " + json.dumps(msg, default=toJson) + "\n") - request.write("\n") + key = [bytes2NativeString(e) for e in event] + msg = dict(key=key, message=data) + request.write(b"event: " + b"event" + b"\n") + request.write( + b"data: " + unicode2bytes(json.dumps(msg, default=toJson)) + b"\n") + request.write(b"\n") def registerQref(self, path, qref): self.qrefs[path] = qref @@ -64,7 +68,7 @@ def __init__(self, master): def decodePath(self, path): for i, p in enumerate(path): - if p == '*': + if p == b'*': path[i] = None return path @@ -75,33 +79,33 @@ def finish(self, request, code, msg): return def render(self, request): - command = "listen" + command = b"listen" path = request.postpath - if path and path[-1] == '': + if path and path[-1] == b'': path = path[:-1] - if path and path[0] in ("listen", "add", "remove"): + if path and path[0] in (b"listen", b"add", b"remove"): command = path[0] path = path[1:] - if command == "listen": - cid = str(uuid.uuid4()) + if command == b"listen": + cid = unicode2bytes(str(uuid.uuid4())) consumer = Consumer(request) - elif command == "add" or command == "remove": + elif command == b"add" or command == b"remove": if path: cid = path[0] path = path[1:] if cid not in self.consumers: - return self.finish(request, 400, "unknown uuid") + return self.finish(request, 400, b"unknown uuid") consumer = self.consumers[cid] else: - return self.finish(request, 400, "need uuid") + return self.finish(request, 400, b"need uuid") - pathref = "/".join(path) + pathref = b"/".join(path) path = self.decodePath(path) - if command == "add" or (command == "listen" and path): + if command == b"add" or (command == b"listen" and path): options = request.args for k in options: if len(options[k]) == 1: @@ -117,22 +121,22 @@ def register(qref): consumer.registerQref(pathref, qref) d.addErrback(log.err, "while calling startConsuming") except NotImplementedError: - return self.finish(request, 404, "not implemented") + return self.finish(request, 404, b"not implemented") except InvalidPathError: - return self.finish(request, 404, "not implemented") - elif command == "remove": + return self.finish(request, 404, b"not implemented") + elif command == b"remove": try: consumer.stopConsuming(pathref) except KeyError: - return self.finish(request, 404, "consumer is not listening to this event") + return self.finish(request, 404, b"consumer is not listening to this event") - if command == "listen": + if command == b"listen": self.consumers[cid] = consumer - request.setHeader("content-type", "text/event-stream") - request.write("") - request.write("event: handshake\n") - request.write("data: " + cid + "\n") - request.write("\n") + request.setHeader(b"content-type", b"text/event-stream") + request.write(b"") + request.write(b"event: handshake\n") + request.write(b"data: " + cid + b"\n") + request.write(b"\n") d = request.notifyFinish() @d.addBoth @@ -142,5 +146,5 @@ def onEndRequest(_): return server.NOT_DONE_YET - self.finish(request, 200, "ok") + self.finish(request, 200, b"ok") return