From 1c86850270b6c9781c0d2f616f720b8cc31ec2cb Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Fri, 24 Jul 2015 15:21:01 -0700 Subject: [PATCH 01/51] Added tchannel.TChannel with new public api --- setup.cfg | 2 +- tchannel/__init__.py | 6 + tchannel/dep/__init__.py | 0 .../scheme.py => dep/thrift_arg_scheme.py} | 14 +-- tchannel/glossary.py | 2 + tchannel/response.py | 19 +++ tchannel/schemes/__init__.py | 24 ++++ tchannel/schemes/json.py | 53 ++++++++ tchannel/schemes/raw.py | 32 +++++ tchannel/schemes/thrift.py | 50 ++++++++ tchannel/tchannel.py | 71 +++++++++++ tchannel/thrift/__init__.py | 1 + tchannel/thrift/client.py | 7 +- tchannel/thrift/module.py | 114 ++++++++++++++++++ tchannel/thrift/serializer.py | 50 ++++++++ tchannel/thrift/server.py | 5 +- tchannel/tornado/broker.py | 6 +- tchannel/tornado/peer.py | 1 + tchannel/transport.py | 41 +++++++ tests/schemes/test_json.py | 59 +++++++++ tests/schemes/test_raw.py | 55 +++++++++ tests/schemes/test_thrift.py | 62 ++++++++++ tests/sync/__init__.py | 0 tests/test_tchannel.py | 67 ++++++++++ tests/thrift/test_module.py | 55 +++++++++ tests/thrift/test_server.py | 10 +- 26 files changed, 782 insertions(+), 24 deletions(-) create mode 100644 tchannel/dep/__init__.py rename tchannel/{thrift/scheme.py => dep/thrift_arg_scheme.py} (88%) create mode 100644 tchannel/response.py create mode 100644 tchannel/schemes/__init__.py create mode 100644 tchannel/schemes/json.py create mode 100644 tchannel/schemes/raw.py create mode 100644 tchannel/schemes/thrift.py create mode 100644 tchannel/tchannel.py create mode 100644 tchannel/thrift/module.py create mode 100644 tchannel/thrift/serializer.py create mode 100644 tchannel/transport.py create mode 100644 tests/schemes/test_json.py create mode 100644 tests/schemes/test_raw.py create mode 100644 tests/schemes/test_thrift.py create mode 100644 tests/sync/__init__.py create mode 100644 tests/test_tchannel.py create mode 100644 tests/thrift/test_module.py diff --git a/setup.cfg b/setup.cfg index 61479714..52960e47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,4 +3,4 @@ create-wheel = yes [flake8] # Ignore files generated by Thrift -exclude = examples/keyvalue/keyvalue/service/*,examples/thrift_examples/hello/*,tchannel/zipkin/thrift/*,tchannel/testing/vcr/proxy/*,tests/data/generated/ThriftTest/* +exclude = examples/keyvalue/keyvalue/service/*,tchannel/zipkin/thrift/*,tchannel/testing/vcr/proxy/*,tests/data/generated/ThriftTest/* diff --git a/tchannel/__init__.py b/tchannel/__init__.py index e69de29b..0c1b8014 100644 --- a/tchannel/__init__.py +++ b/tchannel/__init__.py @@ -0,0 +1,6 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +from .tchannel import TChannel # noqa +from .thrift import from_thrift_module # noqa diff --git a/tchannel/dep/__init__.py b/tchannel/dep/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tchannel/thrift/scheme.py b/tchannel/dep/thrift_arg_scheme.py similarity index 88% rename from tchannel/thrift/scheme.py rename to tchannel/dep/thrift_arg_scheme.py index cfa007da..476468df 100644 --- a/tchannel/thrift/scheme.py +++ b/tchannel/dep/thrift_arg_scheme.py @@ -23,12 +23,12 @@ from thrift.protocol import TBinaryProtocol from thrift.transport import TTransport -from .. import io -from .. import rw -from .. import scheme +from tchannel import io +from tchannel import rw +from tchannel import scheme -class ThriftArgScheme(scheme.ArgScheme): +class DeprecatedThriftArgScheme(scheme.ArgScheme): """Represents the ``thrift`` arg scheme. It requires a reference to the result type for deserialized objects. @@ -42,12 +42,6 @@ class ThriftArgScheme(scheme.ArgScheme): ) def __init__(self, deserialize_type): - """Initialize a new ThriftArgScheme. - - :param deserialize_type: - Type of Thrift object contained in the body. This object will be - deserialized from the stream. - """ self.deserialize_type = deserialize_type def type(self): diff --git a/tchannel/glossary.py b/tchannel/glossary.py index 48c7423a..6a9dbcdd 100644 --- a/tchannel/glossary.py +++ b/tchannel/glossary.py @@ -27,3 +27,5 @@ DEFAULT_TTL = 1000 # ms MAX_ATTEMPT_TIMES = 3 RETRY_DELAY = 0.3 # 300 ms + +DEFAULT_TIMEOUT = DEFAULT_TTL diff --git a/tchannel/response.py b/tchannel/response.py new file mode 100644 index 00000000..9a021dc8 --- /dev/null +++ b/tchannel/response.py @@ -0,0 +1,19 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +from . import transport + +__all__ = ['Response'] + + +class Response(object): + def __init__(self, header, body, transport): + self.header = header + self.body = body + self.transport = transport + + +class ResponseTransportHeaders(transport.TransportHeaders): + """Response-specific Transport Headers""" + pass diff --git a/tchannel/schemes/__init__.py b/tchannel/schemes/__init__.py new file mode 100644 index 00000000..9acc152b --- /dev/null +++ b/tchannel/schemes/__init__.py @@ -0,0 +1,24 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +RAW = 'raw' +JSON = 'json' +THRIFT = 'thrift' +DEFAULT = RAW + +DEFAULT_NAMES = ( + RAW, + JSON, + THRIFT +) + +from .raw import RawArgScheme # noqa +from .json import JsonArgScheme # noqa +from .thrift import ThriftArgScheme # noqa + +DEFAULT_SCHEMES = ( + RawArgScheme, + JsonArgScheme, + ThriftArgScheme +) diff --git a/tchannel/schemes/json.py b/tchannel/schemes/json.py new file mode 100644 index 00000000..4fdd4692 --- /dev/null +++ b/tchannel/schemes/json.py @@ -0,0 +1,53 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +import json + +from tornado import gen + +from . import JSON + + +class JsonArgScheme(object): + """Semantic params and serialization for json.""" + + NAME = JSON + + def __init__(self, tchannel): + self._tchannel = tchannel + + @gen.coroutine + def __call__(self, service, endpoint, body=None, + header=None, timeout=None): + + if header is None: + header = {} + + if body is None: + body = {} + + # serialize + header = json.dumps(header) + body = json.dumps(body) + + response = yield self._tchannel.call( + scheme=self.NAME, + service=service, + arg1=endpoint, + arg2=header, + arg3=body, + timeout=timeout, + ) + + # deserialize + response.header = json.loads(response.header) + response.body = json.loads(response.body) + + raise gen.Return(response) + + def stream(self): + pass + + def register(self): + pass diff --git a/tchannel/schemes/raw.py b/tchannel/schemes/raw.py new file mode 100644 index 00000000..d03989ba --- /dev/null +++ b/tchannel/schemes/raw.py @@ -0,0 +1,32 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +from . import RAW + + +class RawArgScheme(object): + """Semantic params and serialization for raw.""" + + NAME = RAW + + def __init__(self, tchannel): + self._tchannel = tchannel + + def __call__(self, service, endpoint, body=None, + header=None, timeout=None): + + return self._tchannel.call( + scheme=self.NAME, + service=service, + arg1=endpoint, + arg2=header, + arg3=body, + timeout=timeout, + ) + + def stream(self): + pass + + def register(self): + pass diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py new file mode 100644 index 00000000..b29ae9a6 --- /dev/null +++ b/tchannel/schemes/thrift.py @@ -0,0 +1,50 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +from tornado import gen + +from . import THRIFT +from tchannel.thrift import serializer + + +class ThriftArgScheme(object): + + NAME = THRIFT + + def __init__(self, tchannel): + self._tchannel = tchannel + + @gen.coroutine + def __call__(self, request=None, header=None, timeout=None, hostport=None): + + # serialize + header = serializer.serialize_headers(headers=header) + body = serializer.serialize_body(call_args=request.call_args) + + response = yield self._tchannel.call( + scheme=self.NAME, + service=request.service, + arg1=request.endpoint, + arg2=header, + arg3=body, + hostport=hostport + ) + + # deserialize + response.header = serializer.deserialize_headers( + headers=response.header + ) + body = serializer.deserialize_body( + body=response.body, + result_type=request.result_type + ) + response.body = body.success + + raise gen.Return(response) + + def stream(self): + pass + + def register(self): + pass diff --git a/tchannel/tchannel.py b/tchannel/tchannel.py new file mode 100644 index 00000000..0fc38a53 --- /dev/null +++ b/tchannel/tchannel.py @@ -0,0 +1,71 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +from tornado import gen + +from . import schemes, transport +from .glossary import DEFAULT_TIMEOUT +from .response import Response, ResponseTransportHeaders +from .tornado import TChannel as DeprecatedTChannel + +__all__ = ['TChannel'] + + +class TChannel(object): + + def __init__(self, name, hostport=None, process_name=None, + known_peers=None, trace=False): + + # until we move everything here, + # lets compose the old tchannel + self._dep_tchannel = DeprecatedTChannel( + name, hostport, process_name, known_peers, trace + ) + + # set arg schemes + self.raw = schemes.RawArgScheme(self) + self.json = schemes.JsonArgScheme(self) + self.thrift = schemes.ThriftArgScheme(self) + + @gen.coroutine + def call(self, scheme, service, arg1, arg2=None, arg3=None, timeout=None, hostport=None): + + # TODO - dont use asserts for public API + assert format, "format is required" + assert service, "service is required" + assert arg1, "arg1 is required" + + # default args + if arg2 is None: + arg2 = "" + if arg3 is None: + arg3 = "" + if timeout is None: + timeout = DEFAULT_TIMEOUT + + # get operation + # TODO lets treat hostport and service as the same param + operation = self._dep_tchannel.request( + service=service, + hostport=hostport or service, + arg_scheme=scheme + ) + + # fire operation + response = yield operation.send( + arg1=arg1, + arg2=arg2, + arg3=arg3, + headers={'as': scheme} # TODO this is nasty... + ) + + # unwrap response + header = yield response.get_header() + body = yield response.get_body() + t = transport.to_kwargs(response.headers) + t = ResponseTransportHeaders(**t) + + result = Response(header, body, t) + + raise gen.Return(result) diff --git a/tchannel/thrift/__init__.py b/tchannel/thrift/__init__.py index bef78dac..186ea838 100644 --- a/tchannel/thrift/__init__.py +++ b/tchannel/thrift/__init__.py @@ -21,4 +21,5 @@ from __future__ import absolute_import from .client import client_for # noqa +from .module import from_thrift_module # noqa from .server import register # noqa diff --git a/tchannel/thrift/client.py b/tchannel/thrift/client.py index b0466d10..1187d58c 100644 --- a/tchannel/thrift/client.py +++ b/tchannel/thrift/client.py @@ -27,8 +27,7 @@ from tornado import gen from tchannel.tornado.broker import ArgSchemeBroker - -from .scheme import ThriftArgScheme +from tchannel.dep.thrift_arg_scheme import DeprecatedThriftArgScheme from .util import get_service_methods # Generated clients will use this base class. @@ -124,7 +123,7 @@ def generate_method(service_module, service_name, method_name): # TODO result_type is None when the method is oneway. # We don't support oneway yet. - arg_scheme = ThriftArgScheme(result_type) + arg_scheme = DeprecatedThriftArgScheme(result_type) result_spec = result_type.thrift_spec # result_spec is a tuple of tuples in the form: # @@ -162,7 +161,7 @@ def send(self, *args, **kwargs): ), endpoint, {}, - call_args, + call_args, # body protocol_headers=self.protocol_headers, traceflag=self.trace ) diff --git a/tchannel/thrift/module.py b/tchannel/thrift/module.py new file mode 100644 index 00000000..6928c1fd --- /dev/null +++ b/tchannel/thrift/module.py @@ -0,0 +1,114 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +import inspect +import types + +from .util import get_service_methods + + +def from_thrift_module(service, thrift_module): + + # start building requests instance + maker = ThriftRequestMaker(service, thrift_module) + + # create methods that mirror thrift client + # and each return ThriftRequest + methods = _create_methods(thrift_module) + + # attach to maker + for name, method in methods.iteritems(): + method = types.MethodType(method, maker, ThriftRequestMaker) + setattr(maker, name, method) + + return maker + + +class ThriftRequestMaker(object): + + def __init__(self, service, thrift_module): + self.service = service + self.thrift_module = thrift_module + + def _make_request(self, method_name, args, kwargs): + + endpoint = self._get_endpoint(method_name) + result_type = self._get_result_type(method_name) + call_args = self._get_call_args(method_name, args, kwargs) + + request = ThriftRequest( + service=self.service, + endpoint=endpoint, + result_type=result_type, + call_args=call_args + ) + + return request + + def _get_endpoint(self, method_name): + + endpoint = '%s::%s' % (self.service, method_name) + + return endpoint + + def _get_args_type(self, method_name): + + args_type = getattr(self.thrift_module, method_name + '_args') + + return args_type + + def _get_result_type(self, method_name): + + result_type = getattr(self.thrift_module, method_name + '_result') + + return result_type + + def _get_call_args(self, method_name, args, kwargs): + + args_type = self._get_args_type(method_name) + + params = inspect.getcallargs( + getattr(self.thrift_module.Iface, method_name), + self, + *args, + **kwargs + ) + params.pop('self') # self is already known + + call_args = args_type() + for name, value in params.items(): + setattr(call_args, name, value) + + return call_args + + +class ThriftRequest(object): + + def __init__(self, service, endpoint, result_type, call_args): + self.service = service + self.endpoint = endpoint + self.result_type = result_type + self.call_args = call_args + + +def _create_methods(thrift_module): + + methods = {} + method_names = get_service_methods(thrift_module.Iface) + + for method_name in method_names: + + method = _create_method(method_name) + methods[method_name] = method + + return methods + + +def _create_method(method_name): + + def method(self, *args, **kwargs): + # TODO switch to __make_request + return self._make_request(method_name, args, kwargs) + + return method diff --git a/tchannel/thrift/serializer.py b/tchannel/thrift/serializer.py new file mode 100644 index 00000000..13ef17ac --- /dev/null +++ b/tchannel/thrift/serializer.py @@ -0,0 +1,50 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +from thrift.protocol import TBinaryProtocol +from thrift.transport import TTransport + +from .. import io +from .. import rw + +_headers_rw = rw.headers( + rw.number(2), + rw.len_prefixed_string(rw.number(2)), + rw.len_prefixed_string(rw.number(2)), +) + + +def serialize_headers(headers): + + result = _headers_rw.write(headers, io.BytesIO()).getvalue() + + return result + + +def deserialize_headers(headers): + + headers = io.BytesIO(headers) + result = _headers_rw.read(headers) + + return result + + +def serialize_body(call_args): + + trans = TTransport.TMemoryBuffer() + proto = TBinaryProtocol.TBinaryProtocolAccelerated(trans) + call_args.write(proto) + result = trans.getvalue() + + return result + + +def deserialize_body(body, result_type): + + trans = TTransport.TMemoryBuffer(body) + proto = TBinaryProtocol.TBinaryProtocolAccelerated(trans) + + result = result_type() + result.read(proto) + return result diff --git a/tchannel/thrift/server.py b/tchannel/thrift/server.py index f7e0d55b..f6c090b8 100644 --- a/tchannel/thrift/server.py +++ b/tchannel/thrift/server.py @@ -25,12 +25,11 @@ from tornado import gen +from tchannel.dep.thrift_arg_scheme import DeprecatedThriftArgScheme from tchannel.tornado.broker import ArgSchemeBroker from tchannel.tornado.request import TransportMetadata from tchannel.tornado.response import StatusCode -from .scheme import ThriftArgScheme - def register(dispatcher, service_module, handler, method=None, service=None): """Registers a Thrift service method with the given RequestDispatcher. @@ -83,7 +82,7 @@ def hello(request, response, tchannel): endpoint = '%s::%s' % (service, method) args_type = getattr(service_module, method + '_args') result_type = getattr(service_module, method + '_result') - broker = ArgSchemeBroker(ThriftArgScheme(args_type)) + broker = ArgSchemeBroker(DeprecatedThriftArgScheme(args_type)) dispatcher.register( endpoint, diff --git a/tchannel/tornado/broker.py b/tchannel/tornado/broker.py index 4d0d49bb..271cf24d 100644 --- a/tchannel/tornado/broker.py +++ b/tchannel/tornado/broker.py @@ -64,7 +64,11 @@ def handle_call(self, req, resp, proxy): return handler(req, resp, proxy) @tornado.gen.coroutine - def send(self, client, endpoint, header, body, + def send(self, + client, # operation? + endpoint, + header, + body, # NOTE body==call_args protocol_headers=None, traceflag=None, attempt_times=None, diff --git a/tchannel/tornado/peer.py b/tchannel/tornado/peer.py index ac28e208..61579e18 100644 --- a/tchannel/tornado/peer.py +++ b/tchannel/tornado/peer.py @@ -600,6 +600,7 @@ def _send(self, connection, req): @gen.coroutine def send_with_retry(self, request, peer, attempt_times, retry_delay): + # black list to record all used peers, so they aren't chosen again. blacklist = set() response = None diff --git a/tchannel/transport.py b/tchannel/transport.py new file mode 100644 index 00000000..8a5110b4 --- /dev/null +++ b/tchannel/transport.py @@ -0,0 +1,41 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +from . import schemes + +CALLER_NAME = "cn" +CLAIM_AT_START = "cas" +CLAIM_AT_FINISH = "caf" +FAILURE_DOMAIN = "fd" +RETRY_FLAGS = "re" +SCHEME = "as" +SHARD_KEY = "sk" +SPECULATIVE_EXE = "se" + + +def to_kwargs(data): + + args = {} + args['caller_name'] = data.get(CALLER_NAME) + args['claim_at_start'] = data.get(CLAIM_AT_START) + args['claim_at_finish'] = data.get(CLAIM_AT_FINISH) + args['failure_domain'] = data.get(FAILURE_DOMAIN) + args['retry_flags'] = data.get(RETRY_FLAGS) + args['scheme'] = data.get(SCHEME) + args['shard_key'] = data.get(SHARD_KEY) + args['speculative_exe'] = data.get(SPECULATIVE_EXE) + + return args + + +class TransportHeaders(object): + """Transport Headers common between Request & Response""" + + def __init__(self, scheme=None, failure_domain=None, **kwargs): + + if scheme is None: + scheme = schemes.DEFAULT + + self.scheme = scheme + self.failure_domain = failure_domain diff --git a/tests/schemes/test_json.py b/tests/schemes/test_json.py new file mode 100644 index 00000000..b8db1222 --- /dev/null +++ b/tests/schemes/test_json.py @@ -0,0 +1,59 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +import pytest +import tornado + +from tchannel import TChannel, schemes +from tchannel import response +from tchannel.tornado import TChannel as DeprecatedTChannel + + +@pytest.mark.gen_test +@pytest.mark.call +def test_call_should_get_response(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register('endpoint', schemes.JSON) + @tornado.gen.coroutine + def endpoint(request, response, proxy): + + header = yield request.get_header() + body = yield request.get_body() + + assert header == {'req': 'header'} + assert body == {'req': 'body'} + + response.write_header({ + 'resp': 'header' + }) + response.write_body({ + 'resp': 'body' + }) + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + resp = yield tchannel.json( + service=server.hostport, + endpoint='endpoint', + header={'req': 'header'}, + body={'req': 'body'}, + ) + + # verify response + assert isinstance(resp, response.Response) + assert resp.header == {'resp': 'header'} + assert resp.body == {'resp': 'body'} + + # verify response transport headers + assert isinstance(resp.transport, response.ResponseTransportHeaders) + assert resp.transport.scheme == schemes.JSON + assert resp.transport.failure_domain is None diff --git a/tests/schemes/test_raw.py b/tests/schemes/test_raw.py new file mode 100644 index 00000000..eec1dcb8 --- /dev/null +++ b/tests/schemes/test_raw.py @@ -0,0 +1,55 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +import pytest +import tornado + +from tchannel import TChannel, schemes +from tchannel import response +from tchannel.tornado import TChannel as DeprecatedTChannel + + +@pytest.mark.gen_test +@pytest.mark.call +def test_call_should_get_response(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register('endpoint', schemes.RAW) + @tornado.gen.coroutine + def endpoint(request, response, proxy): + + header = yield request.get_header() + body = yield request.get_body() + + assert header == 'raw req header' + assert body == 'raw req body' + + response.write_header('raw resp header') + response.write_body('raw resp body') + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + resp = yield tchannel.raw( + service=server.hostport, + endpoint='endpoint', + header='raw req header', + body='raw req body' + ) + + # verify response + assert isinstance(resp, response.Response) + assert resp.header == 'raw resp header' + assert resp.body == 'raw resp body' + + # verify response transport headers + assert isinstance(resp.transport, response.ResponseTransportHeaders) + assert resp.transport.scheme == schemes.RAW + assert resp.transport.failure_domain is None diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py new file mode 100644 index 00000000..6c398299 --- /dev/null +++ b/tests/schemes/test_thrift.py @@ -0,0 +1,62 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +import pytest + +from tchannel import ( + TChannel, from_thrift_module, + schemes, response +) +from tchannel.tornado import TChannel as DeprecatedTChannel +from tests.data.generated.ThriftTest import ThriftTest + + +@pytest.mark.gen_test +@pytest.mark.call +def test_call_should_get_response(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testStruct(request, response, proxy): + + # assert request.header == {'req': 'header'} + assert request.args.thing.string_thing == 'req string' + + # response.write_header({ + # 'resp': 'header' + # }) + + return ThriftTest.Xtruct( + string_thing="resp string" + ) + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='ThriftTest', # TODO service name ThriftTest is so confusing + thrift_module=ThriftTest + ) + + resp = yield tchannel.thrift( + request=service.testStruct(ThriftTest.Xtruct("req string")), + header={'req': 'header'}, + hostport=server.hostport, # TODO this shouldn't be necessary? + ) + + # verify response + assert isinstance(resp, response.Response) + # assert resp.header == {'resp': 'header'} + assert resp.body == ThriftTest.Xtruct("resp string") + + # verify response transport headers + assert isinstance(resp.transport, response.ResponseTransportHeaders) + assert resp.transport.scheme == schemes.THRIFT + assert resp.transport.failure_domain is None diff --git a/tests/sync/__init__.py b/tests/sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_tchannel.py b/tests/test_tchannel.py new file mode 100644 index 00000000..50916bb0 --- /dev/null +++ b/tests/test_tchannel.py @@ -0,0 +1,67 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +import pytest +import tornado + +from tchannel import TChannel, schemes +from tchannel import response +from tchannel.tornado import TChannel as DeprecatedTChannel + + +@pytest.mark.call +def test_should_have_default_schemes(): + + tchannel = TChannel(name='test') + + for f in schemes.DEFAULT_SCHEMES: + scheme = getattr(tchannel, f.NAME) + assert scheme, "default scheme not found" + assert isinstance(scheme, f) + + +@pytest.mark.gen_test +@pytest.mark.call +def test_call_should_get_response(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register('endpoint', schemes.RAW) + @tornado.gen.coroutine + def endpoint(request, response, proxy): + + header = yield request.get_header() + body = yield request.get_body() + + assert header == 'raw req header' + assert body == 'raw req body' + + response.write_header('raw resp header') + response.write_body('raw resp body') + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + resp = yield tchannel.call( + scheme=schemes.RAW, + service=server.hostport, + arg1='endpoint', + arg2='raw req header', + arg3='raw req body' + ) + + # verify response + assert isinstance(resp, response.Response) + assert resp.header == 'raw resp header' + assert resp.body == 'raw resp body' + + # verify response transport headers + assert isinstance(resp.transport, response.ResponseTransportHeaders) + assert resp.transport.scheme == schemes.RAW + assert resp.transport.failure_domain is None diff --git a/tests/thrift/test_module.py b/tests/thrift/test_module.py new file mode 100644 index 00000000..47879404 --- /dev/null +++ b/tests/thrift/test_module.py @@ -0,0 +1,55 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +import inspect + +import pytest + +from tchannel import from_thrift_module +from tchannel.thrift.module import ThriftRequestMaker, ThriftRequest +from tests.data.generated.ThriftTest import ThriftTest + + +@pytest.mark.call +def test_from_thrift_class_should_return_request_maker(): + + maker = from_thrift_module('thrift_test', ThriftTest) + + assert isinstance(maker, ThriftRequestMaker) + + +@pytest.mark.call +def test_maker_should_have_thrift_iface_methods(): + + # TODO rename ThriftTest to less confusing module name, all lowercase + maker = from_thrift_module('thrift_test', ThriftTest) + + # extract list of maker methods + maker_methods = [ + m[0] for m in + inspect.getmembers(maker, predicate=inspect.ismethod) + ] + + # extract list of iface methods + iface_methods = [ + m[0] for m in + inspect.getmembers(ThriftTest.Iface, predicate=inspect.ismethod) + ] + + # verify all of iface_methods exist in maker_methods + assert set(iface_methods) < set(maker_methods) + + +@pytest.mark.call +def test_request_maker_should_return_request(): + + maker = from_thrift_module('thrift_test', ThriftTest) + + request = maker.testString('hi') + + assert isinstance(request, ThriftRequest) + assert request.service == 'thrift_test' + assert request.endpoint == 'thrift_test::testString' + assert request.result_type == ThriftTest.testString_result + assert request.call_args == ThriftTest.testString_args(thing='hi') diff --git a/tests/thrift/test_server.py b/tests/thrift/test_server.py index 068104fa..8e566721 100644 --- a/tests/thrift/test_server.py +++ b/tests/thrift/test_server.py @@ -24,7 +24,7 @@ import pytest from thrift.Thrift import TType -from tchannel.thrift.scheme import ThriftArgScheme +from tchannel.dep.thrift_arg_scheme import DeprecatedThriftArgScheme from tchannel.thrift.server import build_handler from tchannel.tornado.request import Request from tchannel.tornado.response import Response @@ -108,7 +108,7 @@ def call(treq, tres, tchan): InMemStream('\00\00'), # no headers InMemStream('\00'), # empty struct ], - scheme=ThriftArgScheme(FakeResult), + scheme=DeprecatedThriftArgScheme(FakeResult), headers={'cn': 'test_caller', 'as': 'thrift'}, ) req.close_argstreams() @@ -119,7 +119,7 @@ def call(treq, tres, tchan): response_header, response_body, ], - scheme=ThriftArgScheme(FakeResult), + scheme=DeprecatedThriftArgScheme(FakeResult), ) tchannel = mock.Mock() @@ -161,7 +161,7 @@ def call(treq, tres, tchan): InMemStream('\00\00'), # no headers InMemStream('\00'), # empty struct ], - scheme=ThriftArgScheme(FakeResult), + scheme=DeprecatedThriftArgScheme(FakeResult), ) req.close_argstreams() @@ -171,7 +171,7 @@ def call(treq, tres, tchan): InMemStream(), response_body, ], - scheme=ThriftArgScheme(FakeResult), + scheme=DeprecatedThriftArgScheme(FakeResult), ) tchannel = mock.Mock() From 2680c338e867c9b0e7d77baae68d10d876179e55 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Fri, 31 Jul 2015 11:11:43 -0700 Subject: [PATCH 02/51] util does not describe a problem area --- tchannel/sync/thrift.py | 2 +- tchannel/thrift/client.py | 2 +- tchannel/thrift/module.py | 2 +- tchannel/thrift/{util.py => reflection.py} | 0 tests/thrift/{test_util.py => test_reflection.py} | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename tchannel/thrift/{util.py => reflection.py} (100%) rename tests/thrift/{test_util.py => test_reflection.py} (95%) diff --git a/tchannel/sync/thrift.py b/tchannel/sync/thrift.py index 53b8b7a9..e751d2aa 100644 --- a/tchannel/sync/thrift.py +++ b/tchannel/sync/thrift.py @@ -20,7 +20,7 @@ from __future__ import absolute_import -from tchannel.thrift.util import get_service_methods +from tchannel.thrift.reflection import get_service_methods from tchannel.thrift.client import client_for as async_client_for diff --git a/tchannel/thrift/client.py b/tchannel/thrift/client.py index 1187d58c..e3c2ce26 100644 --- a/tchannel/thrift/client.py +++ b/tchannel/thrift/client.py @@ -28,7 +28,7 @@ from tchannel.tornado.broker import ArgSchemeBroker from tchannel.dep.thrift_arg_scheme import DeprecatedThriftArgScheme -from .util import get_service_methods +from .reflection import get_service_methods # Generated clients will use this base class. _ClientBase = namedtuple( diff --git a/tchannel/thrift/module.py b/tchannel/thrift/module.py index 6928c1fd..4ec73b25 100644 --- a/tchannel/thrift/module.py +++ b/tchannel/thrift/module.py @@ -5,7 +5,7 @@ import inspect import types -from .util import get_service_methods +from .reflection import get_service_methods def from_thrift_module(service, thrift_module): diff --git a/tchannel/thrift/util.py b/tchannel/thrift/reflection.py similarity index 100% rename from tchannel/thrift/util.py rename to tchannel/thrift/reflection.py diff --git a/tests/thrift/test_util.py b/tests/thrift/test_reflection.py similarity index 95% rename from tests/thrift/test_util.py rename to tests/thrift/test_reflection.py index fa8ae99e..9193d194 100644 --- a/tests/thrift/test_util.py +++ b/tests/thrift/test_reflection.py @@ -20,7 +20,7 @@ from __future__ import absolute_import -from tchannel.thrift.util import get_service_methods +from tchannel.thrift.reflection import get_service_methods def test_get_service_methods(): From 2d9868ebd3bd3f320417802134f71ae8fe49eb60 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Fri, 31 Jul 2015 11:23:43 -0700 Subject: [PATCH 03/51] always refer to application headers as headers in client --- tchannel/response.py | 4 ++-- tchannel/schemes/json.py | 12 ++++++------ tchannel/schemes/raw.py | 4 ++-- tchannel/schemes/thrift.py | 10 +++++----- tests/schemes/test_json.py | 10 +++++----- tests/schemes/test_raw.py | 10 +++++----- tests/schemes/test_thrift.py | 8 ++++---- tests/test_tchannel.py | 10 +++++----- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/tchannel/response.py b/tchannel/response.py index 9a021dc8..a3148bac 100644 --- a/tchannel/response.py +++ b/tchannel/response.py @@ -8,8 +8,8 @@ class Response(object): - def __init__(self, header, body, transport): - self.header = header + def __init__(self, headers, body, transport): + self.headers = headers self.body = body self.transport = transport diff --git a/tchannel/schemes/json.py b/tchannel/schemes/json.py index 4fdd4692..f0ff2552 100644 --- a/tchannel/schemes/json.py +++ b/tchannel/schemes/json.py @@ -19,29 +19,29 @@ def __init__(self, tchannel): @gen.coroutine def __call__(self, service, endpoint, body=None, - header=None, timeout=None): + headers=None, timeout=None): - if header is None: - header = {} + if headers is None: + headers = {} if body is None: body = {} # serialize - header = json.dumps(header) + headers = json.dumps(headers) body = json.dumps(body) response = yield self._tchannel.call( scheme=self.NAME, service=service, arg1=endpoint, - arg2=header, + arg2=headers, arg3=body, timeout=timeout, ) # deserialize - response.header = json.loads(response.header) + response.headers = json.loads(response.headers) response.body = json.loads(response.body) raise gen.Return(response) diff --git a/tchannel/schemes/raw.py b/tchannel/schemes/raw.py index d03989ba..09ab40d2 100644 --- a/tchannel/schemes/raw.py +++ b/tchannel/schemes/raw.py @@ -14,13 +14,13 @@ def __init__(self, tchannel): self._tchannel = tchannel def __call__(self, service, endpoint, body=None, - header=None, timeout=None): + headers=None, timeout=None): return self._tchannel.call( scheme=self.NAME, service=service, arg1=endpoint, - arg2=header, + arg2=headers, arg3=body, timeout=timeout, ) diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index b29ae9a6..00620651 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -16,24 +16,24 @@ def __init__(self, tchannel): self._tchannel = tchannel @gen.coroutine - def __call__(self, request=None, header=None, timeout=None, hostport=None): + def __call__(self, request=None, headers=None, timeout=None, hostport=None): # TODO remove hostport here... # serialize - header = serializer.serialize_headers(headers=header) + headers = serializer.serialize_headers(headers=headers) body = serializer.serialize_body(call_args=request.call_args) response = yield self._tchannel.call( scheme=self.NAME, service=request.service, arg1=request.endpoint, - arg2=header, + arg2=headers, arg3=body, hostport=hostport ) # deserialize - response.header = serializer.deserialize_headers( - headers=response.header + response.headers = serializer.deserialize_headers( + headers=response.headers ) body = serializer.deserialize_body( body=response.body, diff --git a/tests/schemes/test_json.py b/tests/schemes/test_json.py index b8db1222..de7c7de3 100644 --- a/tests/schemes/test_json.py +++ b/tests/schemes/test_json.py @@ -22,14 +22,14 @@ def test_call_should_get_response(): @tornado.gen.coroutine def endpoint(request, response, proxy): - header = yield request.get_header() + headers = yield request.get_header() body = yield request.get_body() - assert header == {'req': 'header'} + assert headers == {'req': 'headers'} assert body == {'req': 'body'} response.write_header({ - 'resp': 'header' + 'resp': 'headers' }) response.write_body({ 'resp': 'body' @@ -44,13 +44,13 @@ def endpoint(request, response, proxy): resp = yield tchannel.json( service=server.hostport, endpoint='endpoint', - header={'req': 'header'}, + headers={'req': 'headers'}, body={'req': 'body'}, ) # verify response assert isinstance(resp, response.Response) - assert resp.header == {'resp': 'header'} + assert resp.headers == {'resp': 'headers'} assert resp.body == {'resp': 'body'} # verify response transport headers diff --git a/tests/schemes/test_raw.py b/tests/schemes/test_raw.py index eec1dcb8..7f70d1fe 100644 --- a/tests/schemes/test_raw.py +++ b/tests/schemes/test_raw.py @@ -22,13 +22,13 @@ def test_call_should_get_response(): @tornado.gen.coroutine def endpoint(request, response, proxy): - header = yield request.get_header() + headers = yield request.get_header() body = yield request.get_body() - assert header == 'raw req header' + assert headers == 'raw req headers' assert body == 'raw req body' - response.write_header('raw resp header') + response.write_header('raw resp headers') response.write_body('raw resp body') server.listen() @@ -40,13 +40,13 @@ def endpoint(request, response, proxy): resp = yield tchannel.raw( service=server.hostport, endpoint='endpoint', - header='raw req header', + headers='raw req headers', body='raw req body' ) # verify response assert isinstance(resp, response.Response) - assert resp.header == 'raw resp header' + assert resp.headers == 'raw resp headers' assert resp.body == 'raw resp body' # verify response transport headers diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 6c398299..29b34c69 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -23,11 +23,11 @@ def test_call_should_get_response(): @server.register(ThriftTest) def testStruct(request, response, proxy): - # assert request.header == {'req': 'header'} + # assert request.headers == {'req': 'headers'} assert request.args.thing.string_thing == 'req string' # response.write_header({ - # 'resp': 'header' + # 'resp': 'headers' # }) return ThriftTest.Xtruct( @@ -47,13 +47,13 @@ def testStruct(request, response, proxy): resp = yield tchannel.thrift( request=service.testStruct(ThriftTest.Xtruct("req string")), - header={'req': 'header'}, + headers={'req': 'headers'}, hostport=server.hostport, # TODO this shouldn't be necessary? ) # verify response assert isinstance(resp, response.Response) - # assert resp.header == {'resp': 'header'} + # assert resp.headers == {'resp': 'headers'} assert resp.body == ThriftTest.Xtruct("resp string") # verify response transport headers diff --git a/tests/test_tchannel.py b/tests/test_tchannel.py index 50916bb0..a6cddfac 100644 --- a/tests/test_tchannel.py +++ b/tests/test_tchannel.py @@ -33,13 +33,13 @@ def test_call_should_get_response(): @tornado.gen.coroutine def endpoint(request, response, proxy): - header = yield request.get_header() + headers = yield request.get_header() body = yield request.get_body() - assert header == 'raw req header' + assert headers == 'raw req headers' assert body == 'raw req body' - response.write_header('raw resp header') + response.write_header('raw resp headers') response.write_body('raw resp body') server.listen() @@ -52,13 +52,13 @@ def endpoint(request, response, proxy): scheme=schemes.RAW, service=server.hostport, arg1='endpoint', - arg2='raw req header', + arg2='raw req headers', arg3='raw req body' ) # verify response assert isinstance(resp, response.Response) - assert resp.header == 'raw resp header' + assert resp.headers == 'raw resp headers' assert resp.body == 'raw resp body' # verify response transport headers From 7bd1b63bc7f59ba569dfbd49cdfade0fd4c02eb3 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Fri, 31 Jul 2015 12:54:00 -0700 Subject: [PATCH 04/51] service can be a hyperbahn service name or hostport for client usecase --- tchannel/schemes/thrift.py | 3 +-- tchannel/tchannel.py | 23 ++++++++++++++++------- tchannel/thrift/module.py | 19 ++++++++++++------- tchannel/thrift/reflection.py | 7 +++++++ tests/schemes/test_thrift.py | 3 +-- tests/thrift/test_module.py | 3 +-- 6 files changed, 38 insertions(+), 20 deletions(-) diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index 00620651..b1b9010e 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -16,7 +16,7 @@ def __init__(self, tchannel): self._tchannel = tchannel @gen.coroutine - def __call__(self, request=None, headers=None, timeout=None, hostport=None): # TODO remove hostport here... + def __call__(self, request=None, headers=None, timeout=None): # serialize headers = serializer.serialize_headers(headers=headers) @@ -28,7 +28,6 @@ def __call__(self, request=None, headers=None, timeout=None, hostport=None): # arg1=request.endpoint, arg2=headers, arg3=body, - hostport=hostport ) # deserialize diff --git a/tchannel/tchannel.py b/tchannel/tchannel.py index 0fc38a53..65f5d3b0 100644 --- a/tchannel/tchannel.py +++ b/tchannel/tchannel.py @@ -29,7 +29,7 @@ def __init__(self, name, hostport=None, process_name=None, self.thrift = schemes.ThriftArgScheme(self) @gen.coroutine - def call(self, scheme, service, arg1, arg2=None, arg3=None, timeout=None, hostport=None): + def call(self, scheme, service, arg1, arg2=None, arg3=None, timeout=None): # TODO - dont use asserts for public API assert format, "format is required" @@ -44,13 +44,17 @@ def call(self, scheme, service, arg1, arg2=None, arg3=None, timeout=None, hostpo if timeout is None: timeout = DEFAULT_TIMEOUT + # build request + request_args = { + 'arg_scheme': scheme + } + if _is_hostport(service): + request_args['hostport'] = service + else: + request_args['service'] = service + # get operation - # TODO lets treat hostport and service as the same param - operation = self._dep_tchannel.request( - service=service, - hostport=hostport or service, - arg_scheme=scheme - ) + operation = self._dep_tchannel.request(**request_args) # fire operation response = yield operation.send( @@ -58,6 +62,7 @@ def call(self, scheme, service, arg1, arg2=None, arg3=None, timeout=None, hostpo arg2=arg2, arg3=arg3, headers={'as': scheme} # TODO this is nasty... + # TODO what about other transport headers? ) # unwrap response @@ -69,3 +74,7 @@ def call(self, scheme, service, arg1, arg2=None, arg3=None, timeout=None, hostpo result = Response(header, body, t) raise gen.Return(result) + + +def _is_hostport(service): + return ':' in service diff --git a/tchannel/thrift/module.py b/tchannel/thrift/module.py index 4ec73b25..5f02224e 100644 --- a/tchannel/thrift/module.py +++ b/tchannel/thrift/module.py @@ -5,19 +5,19 @@ import inspect import types -from .reflection import get_service_methods +from .reflection import get_service_methods, get_module_name -def from_thrift_module(service, thrift_module): +def from_thrift_module(service, thrift_module, thrift_class_name=None): - # start building requests instance - maker = ThriftRequestMaker(service, thrift_module) + # start with a request maker instance + maker = ThriftRequestMaker(service, thrift_module, thrift_class_name) # create methods that mirror thrift client # and each return ThriftRequest methods = _create_methods(thrift_module) - # attach to maker + # then attach to instane for name, method in methods.iteritems(): method = types.MethodType(method, maker, ThriftRequestMaker) setattr(maker, name, method) @@ -27,10 +27,15 @@ def from_thrift_module(service, thrift_module): class ThriftRequestMaker(object): - def __init__(self, service, thrift_module): + def __init__(self, service, thrift_module, thrift_class_name=None): self.service = service self.thrift_module = thrift_module + if thrift_class_name is not None: + self.thrift_class_name = thrift_class_name + else: + self.thrift_class_name = get_module_name(self.thrift_module) + def _make_request(self, method_name, args, kwargs): endpoint = self._get_endpoint(method_name) @@ -48,7 +53,7 @@ def _make_request(self, method_name, args, kwargs): def _get_endpoint(self, method_name): - endpoint = '%s::%s' % (self.service, method_name) + endpoint = '%s::%s' % (self.thrift_class_name, method_name) return endpoint diff --git a/tchannel/thrift/reflection.py b/tchannel/thrift/reflection.py index cf947f87..b1dc1203 100644 --- a/tchannel/thrift/reflection.py +++ b/tchannel/thrift/reflection.py @@ -38,3 +38,10 @@ def get_service_methods(iface): return set( name for (name, method) in methods if not name.startswith('__') ) + + +def get_module_name(module): + + name = module.__name__.rsplit('.', 1)[-1] + + return name diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 29b34c69..993609ad 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -41,14 +41,13 @@ def testStruct(request, response, proxy): tchannel = TChannel(name='client') service = from_thrift_module( - service='ThriftTest', # TODO service name ThriftTest is so confusing + service=server.hostport, thrift_module=ThriftTest ) resp = yield tchannel.thrift( request=service.testStruct(ThriftTest.Xtruct("req string")), headers={'req': 'headers'}, - hostport=server.hostport, # TODO this shouldn't be necessary? ) # verify response diff --git a/tests/thrift/test_module.py b/tests/thrift/test_module.py index 47879404..4e26b06b 100644 --- a/tests/thrift/test_module.py +++ b/tests/thrift/test_module.py @@ -22,7 +22,6 @@ def test_from_thrift_class_should_return_request_maker(): @pytest.mark.call def test_maker_should_have_thrift_iface_methods(): - # TODO rename ThriftTest to less confusing module name, all lowercase maker = from_thrift_module('thrift_test', ThriftTest) # extract list of maker methods @@ -50,6 +49,6 @@ def test_request_maker_should_return_request(): assert isinstance(request, ThriftRequest) assert request.service == 'thrift_test' - assert request.endpoint == 'thrift_test::testString' + assert request.endpoint == 'ThriftTest::testString' assert request.result_type == ThriftTest.testString_result assert request.call_args == ThriftTest.testString_args(thing='hi') From 23f87f4ef005493d7a60e7537e957cfba40c1fed Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Fri, 31 Jul 2015 13:16:35 -0700 Subject: [PATCH 05/51] thrift call-site headers --- tchannel/thrift/serializer.py | 7 ++++++- tests/schemes/test_thrift.py | 12 +++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tchannel/thrift/serializer.py b/tchannel/thrift/serializer.py index 13ef17ac..12dcb4dc 100644 --- a/tchannel/thrift/serializer.py +++ b/tchannel/thrift/serializer.py @@ -25,7 +25,12 @@ def serialize_headers(headers): def deserialize_headers(headers): headers = io.BytesIO(headers) - result = _headers_rw.read(headers) + headers = _headers_rw.read(headers) + + result = {} + + for h in headers: + result[h[0]] = h[1] return result diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 993609ad..68d7acf2 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -23,12 +23,18 @@ def test_call_should_get_response(): @server.register(ThriftTest) def testStruct(request, response, proxy): + # TODO server getting headers in non-friendly format, + # create a top-level request that has friendly headers :) # assert request.headers == {'req': 'headers'} + assert request.headers == [['req', 'header']] assert request.args.thing.string_thing == 'req string' + # TODO should this response object be shared w client case? + # TODO are we ok with the approach here? it's diff than client... # response.write_header({ - # 'resp': 'headers' + # 'resp': 'header' # }) + response.write_header('resp', 'header') return ThriftTest.Xtruct( string_thing="resp string" @@ -47,12 +53,12 @@ def testStruct(request, response, proxy): resp = yield tchannel.thrift( request=service.testStruct(ThriftTest.Xtruct("req string")), - headers={'req': 'headers'}, + headers={'req': 'header'}, ) # verify response assert isinstance(resp, response.Response) - # assert resp.headers == {'resp': 'headers'} + assert resp.headers == {'resp': 'header'} assert resp.body == ThriftTest.Xtruct("resp string") # verify response transport headers From b3349c5077584a1579e8ccc47130a24178b50e37 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Fri, 31 Jul 2015 14:54:33 -0700 Subject: [PATCH 06/51] add timeout & retry directives to call --- tchannel/glossary.py | 6 +---- tchannel/messages/call_request.py | 4 ++-- tchannel/messages/cancel.py | 4 ++-- tchannel/messages/claim.py | 4 ++-- tchannel/retry.py | 12 ++++++++++ tchannel/schemes/json.py | 6 +++-- tchannel/schemes/raw.py | 6 +++-- tchannel/schemes/thrift.py | 6 ++++- tchannel/tchannel.py | 40 +++++++++++++++++++++---------- tchannel/tornado/peer.py | 28 ++++++++++++---------- tchannel/tornado/request.py | 14 +++++------ tchannel/transport_header.py | 39 ------------------------------ tests/integration/test_retry.py | 39 +++++++++++++++--------------- 13 files changed, 102 insertions(+), 106 deletions(-) create mode 100644 tchannel/retry.py delete mode 100644 tchannel/transport_header.py diff --git a/tchannel/glossary.py b/tchannel/glossary.py index 6a9dbcdd..36bff036 100644 --- a/tchannel/glossary.py +++ b/tchannel/glossary.py @@ -24,8 +24,4 @@ MAX_MESSAGE_ID = 0xfffffffe # CallRequestMessage uses it as the default TTL value for the message. -DEFAULT_TTL = 1000 # ms -MAX_ATTEMPT_TIMES = 3 -RETRY_DELAY = 0.3 # 300 ms - -DEFAULT_TIMEOUT = DEFAULT_TTL +DEFAULT_TIMEOUT = 1000 # ms diff --git a/tchannel/messages/call_request.py b/tchannel/messages/call_request.py index b1a56344..1fe0792d 100644 --- a/tchannel/messages/call_request.py +++ b/tchannel/messages/call_request.py @@ -22,7 +22,7 @@ from . import common from .. import rw -from ..glossary import DEFAULT_TTL +from ..glossary import DEFAULT_TIMEOUT from .call_request_continue import CallRequestContinueMessage from .types import Types @@ -41,7 +41,7 @@ class CallRequestMessage(CallRequestContinueMessage): def __init__( self, flags=0, - ttl=DEFAULT_TTL, + ttl=DEFAULT_TIMEOUT, tracing=None, service=None, headers=None, diff --git a/tchannel/messages/cancel.py b/tchannel/messages/cancel.py index db5a2594..c36f04df 100644 --- a/tchannel/messages/cancel.py +++ b/tchannel/messages/cancel.py @@ -22,7 +22,7 @@ from . import common from .. import rw -from ..glossary import DEFAULT_TTL +from ..glossary import DEFAULT_TIMEOUT from .base import BaseMessage @@ -33,7 +33,7 @@ class CancelMessage(BaseMessage): 'why', ) - def __init__(self, ttl=DEFAULT_TTL, tracing=None, why=None, id=0): + def __init__(self, ttl=DEFAULT_TIMEOUT, tracing=None, why=None, id=0): super(CancelMessage, self).__init__(id) self.ttl = ttl self.tracing = tracing or common.Tracing(0, 0, 0, 0) diff --git a/tchannel/messages/claim.py b/tchannel/messages/claim.py index f5f0d2f5..5400a09e 100644 --- a/tchannel/messages/claim.py +++ b/tchannel/messages/claim.py @@ -22,7 +22,7 @@ from . import common from .. import rw -from ..glossary import DEFAULT_TTL +from ..glossary import DEFAULT_TIMEOUT from .base import BaseMessage @@ -32,7 +32,7 @@ class ClaimMessage(BaseMessage): 'tracing', ) - def __init__(self, ttl=DEFAULT_TTL, tracing=None, id=0): + def __init__(self, ttl=DEFAULT_TIMEOUT, tracing=None, id=0): super(ClaimMessage, self).__init__(id) self.ttl = ttl self.tracing = tracing or common.Tracing(0, 0, 0, 0) diff --git a/tchannel/retry.py b/tchannel/retry.py new file mode 100644 index 00000000..ea701a10 --- /dev/null +++ b/tchannel/retry.py @@ -0,0 +1,12 @@ +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +NEVER = 'n' +CONNECTION_ERROR = 'c' +TIMEOUT = 't' +CONNECTION_ERROR_AND_TIMEOUT = 'ct' +DEFAULT = CONNECTION_ERROR + +DEFAULT_RETRY_LIMIT = 3 +DEFAULT_RETRY_DELAY = 0.3 # 300 ms diff --git a/tchannel/schemes/json.py b/tchannel/schemes/json.py index f0ff2552..d752b50f 100644 --- a/tchannel/schemes/json.py +++ b/tchannel/schemes/json.py @@ -18,8 +18,8 @@ def __init__(self, tchannel): self._tchannel = tchannel @gen.coroutine - def __call__(self, service, endpoint, body=None, - headers=None, timeout=None): + def __call__(self, service, endpoint, body=None, headers=None, + timeout=None, retry_on=None, retry_limit=None): if headers is None: headers = {} @@ -38,6 +38,8 @@ def __call__(self, service, endpoint, body=None, arg2=headers, arg3=body, timeout=timeout, + retry_on=retry_on, + retry_limit=retry_limit ) # deserialize diff --git a/tchannel/schemes/raw.py b/tchannel/schemes/raw.py index 09ab40d2..f5d3d028 100644 --- a/tchannel/schemes/raw.py +++ b/tchannel/schemes/raw.py @@ -13,8 +13,8 @@ class RawArgScheme(object): def __init__(self, tchannel): self._tchannel = tchannel - def __call__(self, service, endpoint, body=None, - headers=None, timeout=None): + def __call__(self, service, endpoint, body=None, headers=None, + timeout=None, retry_on=None, retry_limit=None): return self._tchannel.call( scheme=self.NAME, @@ -23,6 +23,8 @@ def __call__(self, service, endpoint, body=None, arg2=headers, arg3=body, timeout=timeout, + retry_on=retry_on, + retry_limit=retry_limit ) def stream(self): diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index b1b9010e..bba35153 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -16,7 +16,8 @@ def __init__(self, tchannel): self._tchannel = tchannel @gen.coroutine - def __call__(self, request=None, headers=None, timeout=None): + def __call__(self, request=None, headers=None, timeout=None, + retry_on=None, retry_limit=None): # serialize headers = serializer.serialize_headers(headers=headers) @@ -28,6 +29,9 @@ def __call__(self, request=None, headers=None, timeout=None): arg1=request.endpoint, arg2=headers, arg3=body, + timeout=timeout, + retry_on=retry_on, + retry_limit=retry_limit ) # deserialize diff --git a/tchannel/tchannel.py b/tchannel/tchannel.py index 65f5d3b0..43f4e24a 100644 --- a/tchannel/tchannel.py +++ b/tchannel/tchannel.py @@ -4,7 +4,7 @@ from tornado import gen -from . import schemes, transport +from . import schemes, transport, retry from .glossary import DEFAULT_TIMEOUT from .response import Response, ResponseTransportHeaders from .tornado import TChannel as DeprecatedTChannel @@ -23,13 +23,16 @@ def __init__(self, name, hostport=None, process_name=None, name, hostport, process_name, known_peers, trace ) + self.name = name + # set arg schemes self.raw = schemes.RawArgScheme(self) self.json = schemes.JsonArgScheme(self) self.thrift = schemes.ThriftArgScheme(self) @gen.coroutine - def call(self, scheme, service, arg1, arg2=None, arg3=None, timeout=None): + def call(self, scheme, service, arg1, arg2=None, arg3=None, + timeout=None, retry_on=None, retry_limit=None): # TODO - dont use asserts for public API assert format, "format is required" @@ -43,26 +46,39 @@ def call(self, scheme, service, arg1, arg2=None, arg3=None, timeout=None): arg3 = "" if timeout is None: timeout = DEFAULT_TIMEOUT - - # build request - request_args = { - 'arg_scheme': scheme + if retry_on is None: + retry_on = retry.DEFAULT + if retry_limit is None: + retry_limit = retry.DEFAULT_RETRY_LIMIT + + # build operation + operation_args = { + 'service': None, + 'arg_scheme': scheme, + 'retry': retry_on, } if _is_hostport(service): - request_args['hostport'] = service + operation_args['hostport'] = service else: - request_args['service'] = service + operation_args['service'] = service - # get operation - operation = self._dep_tchannel.request(**request_args) + # calls tchannel.tornado.peer.PeerClientOperation.__init__ + operation = self._dep_tchannel.request(**operation_args) # fire operation + + transport_headers = { + transport.SCHEME: scheme, + transport.CALLER_NAME: self.name, + } + response = yield operation.send( arg1=arg1, arg2=arg2, arg3=arg3, - headers={'as': scheme} # TODO this is nasty... - # TODO what about other transport headers? + headers=transport_headers, + attempt_times=retry_limit, + ttl=timeout ) # unwrap response diff --git a/tchannel/tornado/peer.py b/tchannel/tornado/peer.py index 61579e18..e1037af1 100644 --- a/tchannel/tornado/peer.py +++ b/tchannel/tornado/peer.py @@ -18,7 +18,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from __future__ import absolute_import +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) import logging from collections import deque @@ -27,17 +29,16 @@ from tornado import gen +from ..schemes import DEFAULT as DEFAULT_SCHEME +from ..retry import ( + DEFAULT as DEFAULT_RETRY, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY +) from tchannel.event import EventType -from tchannel.glossary import DEFAULT_TTL -from tchannel.glossary import MAX_ATTEMPT_TIMES -from tchannel.glossary import RETRY_DELAY - +from tchannel.glossary import DEFAULT_TIMEOUT from ..errors import NoAvailablePeerError from ..errors import ProtocolError from ..errors import TimeoutError from ..handler import CallableRequestHandler -from ..transport_header import ArgSchemeType -from ..transport_header import RetryType from ..zipkin.annotation import Endpoint from ..zipkin.trace import Trace from .connection import StreamConnection @@ -451,9 +452,12 @@ def __init__(self, peer, service, arg_scheme=None, self.peer = peer self.service = service self.parent_tracing = parent_tracing + + # TODO the term headers are reserved for application headers, + # these are transport headers, self.headers = { - 'as': arg_scheme or ArgSchemeType.DEFAULT, - 're': retry or RetryType.DEFAULT, + 'as': arg_scheme or DEFAULT_SCHEME, + 're': retry or DEFAULT_RETRY, 'cn': self.peer.tchannel.name, } @@ -501,9 +505,9 @@ def send(self, arg1, arg2, arg3, maybe_stream(arg1), maybe_stream(arg2), maybe_stream(arg3) ) - attempt_times = attempt_times or MAX_ATTEMPT_TIMES - ttl = ttl or DEFAULT_TTL - retry_delay = retry_delay or RETRY_DELAY + attempt_times = attempt_times or DEFAULT_RETRY_LIMIT + ttl = ttl or DEFAULT_TIMEOUT + retry_delay = retry_delay or DEFAULT_RETRY_DELAY # hack to get endpoint from arg_1 for trace name arg1.close() endpoint = yield read_full(arg1) diff --git a/tchannel/tornado/request.py b/tchannel/tornado/request.py index 23903208..59bbf321 100644 --- a/tchannel/tornado/request.py +++ b/tchannel/tornado/request.py @@ -25,11 +25,11 @@ import tornado import tornado.gen -from ..glossary import DEFAULT_TTL +from tchannel import retry +from ..glossary import DEFAULT_TIMEOUT from ..messages import ErrorCode from ..messages.common import FlagsType from ..messages.common import StreamState -from ..transport_header import RetryType from ..zipkin.trace import Trace from .stream import InMemStream from .util import get_arg @@ -47,7 +47,7 @@ def __init__( self, id=None, flags=FlagsType.none, - ttl=DEFAULT_TTL, + ttl=DEFAULT_TIMEOUT, tracing=None, service=None, headers=None, @@ -167,9 +167,9 @@ def should_retry_on_error(self, error): # not retry for streaming request return False - retry_flag = self.headers.get('re', RetryType.DEFAULT) + retry_flag = self.headers.get('re', retry.DEFAULT) - if retry_flag == RetryType.NEVER: + if retry_flag == retry.NEVER: return False if error.code in [ErrorCode.bad_request, ErrorCode.cancelled, @@ -178,11 +178,11 @@ def should_retry_on_error(self, error): elif error.code in [ErrorCode.busy, ErrorCode.declined]: return True elif error.code is ErrorCode.timeout: - return retry_flag is not RetryType.CONNECTION_ERROR + return retry_flag is not retry.CONNECTION_ERROR elif error.code in [ErrorCode.network_error, ErrorCode.fatal, ErrorCode.unexpected]: - return retry_flag is not RetryType.TIMEOUT + return retry_flag is not retry.TIMEOUT else: return False diff --git a/tchannel/transport_header.py b/tchannel/transport_header.py deleted file mode 100644 index 6f5b32be..00000000 --- a/tchannel/transport_header.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - - -class RetryType(object): - """ Retry Type in the protocol header - For details, look at protocol definition. - https://github.com/uber/tchannel/blob/master/docs/protocol.md - """ - NEVER = 'n' - CONNECTION_ERROR = 'c' - TIMEOUT = 't' - CONNECTION_ERROR_AND_TIMEOUT = 'ct' - DEFAULT = CONNECTION_ERROR - - -class ArgSchemeType(object): - RAW = 'raw' - JSON = 'json' - HTTP = 'http' - THRIFT = 'thrift' - DEFAULT = RAW diff --git a/tests/integration/test_retry.py b/tests/integration/test_retry.py index d8acdb43..daec8afe 100644 --- a/tests/integration/test_retry.py +++ b/tests/integration/test_retry.py @@ -25,15 +25,14 @@ import tornado.gen from mock import patch +from tchannel import retry from tchannel.errors import ProtocolError from tchannel.errors import TChannelError from tchannel.errors import TimeoutError -from tchannel.glossary import MAX_ATTEMPT_TIMES from tchannel.messages import ErrorCode from tchannel.tornado import Request from tchannel.tornado import TChannel from tchannel.tornado.stream import InMemStream -from tchannel.transport_header import RetryType @tornado.gen.coroutine @@ -86,7 +85,7 @@ def test_retry_timeout(): "test", "test", headers={ - 're': RetryType.CONNECTION_ERROR_AND_TIMEOUT + 're': retry.CONNECTION_ERROR_AND_TIMEOUT }, ttl=0.005, attempt_times=3, @@ -113,7 +112,7 @@ def test_retry_on_error_fail(): "test", "test", headers={ - 're': RetryType.CONNECTION_ERROR_AND_TIMEOUT + 're': retry.CONNECTION_ERROR_AND_TIMEOUT }, ttl=0.02, attempt_times=3, @@ -122,7 +121,7 @@ def test_retry_on_error_fail(): assert mock_should_retry_on_error.called assert mock_should_retry_on_error.call_count == ( - MAX_ATTEMPT_TIMES) + retry.DEFAULT_RETRY_LIMIT) assert e.value.code == ErrorCode.busy @@ -150,7 +149,7 @@ def test_retry_on_error_success(): "test", "test", headers={ - 're': RetryType.CONNECTION_ERROR_AND_TIMEOUT, + 're': retry.CONNECTION_ERROR_AND_TIMEOUT, }, ttl=0.01, attempt_times=3, @@ -165,20 +164,20 @@ def test_retry_on_error_success(): @pytest.mark.gen_test @pytest.mark.parametrize('retry_flag, error_code, result', [ - (RetryType.CONNECTION_ERROR, ErrorCode.busy, True), - (RetryType.CONNECTION_ERROR, ErrorCode.declined, True), - (RetryType.CONNECTION_ERROR, ErrorCode.timeout, False), - (RetryType.CONNECTION_ERROR_AND_TIMEOUT, ErrorCode.timeout, True), - (RetryType.TIMEOUT, ErrorCode.unexpected, False), - (RetryType.TIMEOUT, ErrorCode.network_error, False), - (RetryType.CONNECTION_ERROR, ErrorCode.network_error, True), - (RetryType.NEVER, ErrorCode.network_error, False), - (RetryType.CONNECTION_ERROR_AND_TIMEOUT, ErrorCode.cancelled, False), - (RetryType.CONNECTION_ERROR_AND_TIMEOUT, ErrorCode.bad_request, False), - (RetryType.CONNECTION_ERROR, ErrorCode.fatal, True), - (RetryType.TIMEOUT, ErrorCode.fatal, False), - (RetryType.TIMEOUT, ErrorCode.unhealthy, False), - (RetryType.CONNECTION_ERROR_AND_TIMEOUT, ErrorCode.unhealthy, False), + (retry.CONNECTION_ERROR, ErrorCode.busy, True), + (retry.CONNECTION_ERROR, ErrorCode.declined, True), + (retry.CONNECTION_ERROR, ErrorCode.timeout, False), + (retry.CONNECTION_ERROR_AND_TIMEOUT, ErrorCode.timeout, True), + (retry.TIMEOUT, ErrorCode.unexpected, False), + (retry.TIMEOUT, ErrorCode.network_error, False), + (retry.CONNECTION_ERROR, ErrorCode.network_error, True), + (retry.NEVER, ErrorCode.network_error, False), + (retry.CONNECTION_ERROR_AND_TIMEOUT, ErrorCode.cancelled, False), + (retry.CONNECTION_ERROR_AND_TIMEOUT, ErrorCode.bad_request, False), + (retry.CONNECTION_ERROR, ErrorCode.fatal, True), + (retry.TIMEOUT, ErrorCode.fatal, False), + (retry.TIMEOUT, ErrorCode.unhealthy, False), + (retry.CONNECTION_ERROR_AND_TIMEOUT, ErrorCode.unhealthy, False), ], ids=lambda arg: str(arg) ) From a7840265805ba3baac2e0155c5fb4d701916d27f Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 13:31:33 -0700 Subject: [PATCH 07/51] remove overloading of service & hostport --- examples/json/server.py | 36 ++++++++++++++++++++++++++++++++++ examples/raw/client.py | 28 ++++++++++++++++++++++++++ examples/raw/server.py | 21 ++++++++++++++++++++ examples/thrift/client.py | 38 ++++++++++++++++++++++++++++++++++++ examples/thrift/server.py | 26 ++++++++++++++++++++++++ exapmles/json/client.py | 35 +++++++++++++++++++++++++++++++++ tchannel/schemes/json.py | 7 +++++-- tchannel/schemes/raw.py | 5 +++-- tchannel/schemes/thrift.py | 3 ++- tchannel/tchannel.py | 24 ++++++++--------------- tchannel/thrift/module.py | 23 +++++++++++++++++----- tests/schemes/test_json.py | 3 ++- tests/schemes/test_raw.py | 5 +++-- tests/schemes/test_thrift.py | 5 +++-- tests/test_tchannel.py | 5 +++-- 15 files changed, 231 insertions(+), 33 deletions(-) create mode 100644 examples/json/server.py create mode 100644 examples/raw/client.py create mode 100644 examples/raw/server.py create mode 100644 examples/thrift/client.py create mode 100644 examples/thrift/server.py create mode 100644 exapmles/json/client.py diff --git a/examples/json/server.py b/examples/json/server.py new file mode 100644 index 00000000..d9ad8f1b --- /dev/null +++ b/examples/json/server.py @@ -0,0 +1,36 @@ +from tornado import gen +from tornado.ioloop import IOLoop + +from tchannel.tornado import TChannel +from tchannel import schemes + + +app = TChannel('json-server') + + +@app.register('health', schemes.JSON) +@gen.coroutine +def health(request, response, tchannel): + + # TODO in thrift you dont have to yield... + body = yield request.get_body() + + # TODO should be get_headers() + headers = yield request.get_header() + + # TODO should be write_headers() + response.write_header(headers) + + # TODO should be able to return body no matter the scheme + response.write_body({ + 'health': 'OK', + 'body': body, + }) + + +app.listen() + +print app.hostport + +io_loop = IOLoop.current() +io_loop.start() diff --git a/examples/raw/client.py b/examples/raw/client.py new file mode 100644 index 00000000..c1d9c8f7 --- /dev/null +++ b/examples/raw/client.py @@ -0,0 +1,28 @@ +from tornado import gen, ioloop + +from tchannel import TChannel + + +client = TChannel('raw-client') + + +@gen.coroutine +def make_request(): + + resp = yield client.raw( + service='10.32.160.131:52250', + endpoint='health' + ) + + raise gen.Return(resp) + + +io_loop = ioloop.IOLoop.current() + +resp = io_loop.run_sync(make_request) + + +# TODO impl __repr__ +print resp.body +print resp.headers +print resp.transport diff --git a/examples/raw/server.py b/examples/raw/server.py new file mode 100644 index 00000000..66f54f59 --- /dev/null +++ b/examples/raw/server.py @@ -0,0 +1,21 @@ +from tornado.ioloop import IOLoop + +from tchannel.tornado import TChannel + + +app = TChannel('raw-server') + + +@app.register('health') +def health(request, response, tchannel): + + # TODO should be able to return body no matter the scheme + response.write_body('OK') + + +app.listen() + +print app.hostport + +io_loop = IOLoop.current() +io_loop.start() diff --git a/examples/thrift/client.py b/examples/thrift/client.py new file mode 100644 index 00000000..f2a85804 --- /dev/null +++ b/examples/thrift/client.py @@ -0,0 +1,38 @@ +from tornado import gen, ioloop + +from tchannel import TChannel, from_thrift_module + +from tests.data.generated.ThriftTest import ThriftTest + + +tchannel = TChannel('thrift-client') + +service = '10.32.160.131:52294' +thrift_client = from_thrift_module(service, ThriftTest) + + +@gen.coroutine +def make_request(): + + resp = yield tchannel.thrift( + request=thrift_client.testString(thing="holler"), + + # TODO bug, you have to have headers :P + headers={'wtf': 'dude'} + ) + + raise gen.Return(resp) + + +io_loop = ioloop.IOLoop.current() + +resp = io_loop.run_sync(make_request) + + +# TODO impl __repr__ +print resp.body + +# TODO wtf tests have headers... +print resp.headers + +print resp.transport diff --git a/examples/thrift/server.py b/examples/thrift/server.py new file mode 100644 index 00000000..e4807650 --- /dev/null +++ b/examples/thrift/server.py @@ -0,0 +1,26 @@ +from tornado import gen +from tornado.ioloop import IOLoop + +from tchannel.tornado import TChannel + +from tests.data.generated.ThriftTest import ThriftTest + +app = TChannel('thrift-server') + + +@app.register(ThriftTest) +@gen.coroutine +def testString(request, response, tchannel): + + # TODO different than other schemes... + response.write_header('hey', 'jane') + + return request.args.thing + + +app.listen() + +print app.hostport + +io_loop = IOLoop.current() +io_loop.start() diff --git a/exapmles/json/client.py b/exapmles/json/client.py new file mode 100644 index 00000000..8261f5e7 --- /dev/null +++ b/exapmles/json/client.py @@ -0,0 +1,35 @@ +from tornado import gen, ioloop + +from tchannel import TChannel + + +client = TChannel('json-client') +service = '10.32.160.131:52275' + + +@gen.coroutine +def make_request(): + + resp = yield client.json( + service=service, + endpoint='health', + body={ + 'boomboomboom': 'hearyousaywayo' + }, + headers={ + 'bobby': 'twotoes' + } + ) + + raise gen.Return(resp) + + +io_loop = ioloop.IOLoop.current() + +resp = io_loop.run_sync(make_request) + + +# TODO impl __repr__ +print resp.body +print resp.headers +print resp.transport diff --git a/tchannel/schemes/json.py b/tchannel/schemes/json.py index d752b50f..265274d5 100644 --- a/tchannel/schemes/json.py +++ b/tchannel/schemes/json.py @@ -19,11 +19,13 @@ def __init__(self, tchannel): @gen.coroutine def __call__(self, service, endpoint, body=None, headers=None, - timeout=None, retry_on=None, retry_limit=None): + timeout=None, retry_on=None, retry_limit=None, hostport=None): + # TODO should we not default these? if headers is None: headers = {} + # TODO dont default? if body is None: body = {} @@ -39,7 +41,8 @@ def __call__(self, service, endpoint, body=None, headers=None, arg3=body, timeout=timeout, retry_on=retry_on, - retry_limit=retry_limit + retry_limit=retry_limit, + hostport=hostport, ) # deserialize diff --git a/tchannel/schemes/raw.py b/tchannel/schemes/raw.py index f5d3d028..019eb3f6 100644 --- a/tchannel/schemes/raw.py +++ b/tchannel/schemes/raw.py @@ -14,7 +14,7 @@ def __init__(self, tchannel): self._tchannel = tchannel def __call__(self, service, endpoint, body=None, headers=None, - timeout=None, retry_on=None, retry_limit=None): + timeout=None, retry_on=None, retry_limit=None, hostport=None): return self._tchannel.call( scheme=self.NAME, @@ -24,7 +24,8 @@ def __call__(self, service, endpoint, body=None, headers=None, arg3=body, timeout=timeout, retry_on=retry_on, - retry_limit=retry_limit + retry_limit=retry_limit, + hostport=hostport, ) def stream(self): diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index bba35153..76438cf3 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -31,7 +31,8 @@ def __call__(self, request=None, headers=None, timeout=None, arg3=body, timeout=timeout, retry_on=retry_on, - retry_limit=retry_limit + retry_limit=retry_limit, + hostport=request.hostport ) # deserialize diff --git a/tchannel/tchannel.py b/tchannel/tchannel.py index 43f4e24a..5b73fbbf 100644 --- a/tchannel/tchannel.py +++ b/tchannel/tchannel.py @@ -32,7 +32,7 @@ def __init__(self, name, hostport=None, process_name=None, @gen.coroutine def call(self, scheme, service, arg1, arg2=None, arg3=None, - timeout=None, retry_on=None, retry_limit=None): + timeout=None, retry_on=None, retry_limit=None, hostport=None): # TODO - dont use asserts for public API assert format, "format is required" @@ -51,34 +51,26 @@ def call(self, scheme, service, arg1, arg2=None, arg3=None, if retry_limit is None: retry_limit = retry.DEFAULT_RETRY_LIMIT - # build operation - operation_args = { - 'service': None, - 'arg_scheme': scheme, - 'retry': retry_on, - } - if _is_hostport(service): - operation_args['hostport'] = service - else: - operation_args['service'] = service - # calls tchannel.tornado.peer.PeerClientOperation.__init__ - operation = self._dep_tchannel.request(**operation_args) + operation = self._dep_tchannel.request( + service=service, + hostport=hostport, + arg_scheme=scheme, + retry=retry_on, + ) # fire operation - transport_headers = { transport.SCHEME: scheme, transport.CALLER_NAME: self.name, } - response = yield operation.send( arg1=arg1, arg2=arg2, arg3=arg3, headers=transport_headers, attempt_times=retry_limit, - ttl=timeout + ttl=timeout, ) # unwrap response diff --git a/tchannel/thrift/module.py b/tchannel/thrift/module.py index 5f02224e..7c7263a9 100644 --- a/tchannel/thrift/module.py +++ b/tchannel/thrift/module.py @@ -8,10 +8,16 @@ from .reflection import get_service_methods, get_module_name -def from_thrift_module(service, thrift_module, thrift_class_name=None): +def from_thrift_module(service, thrift_module, hostport=None, + thrift_class_name=None): # start with a request maker instance - maker = ThriftRequestMaker(service, thrift_module, thrift_class_name) + maker = ThriftRequestMaker( + service=service, + thrift_module=thrift_module, + hostport=hostport, + thrift_class_name=thrift_class_name + ) # create methods that mirror thrift client # and each return ThriftRequest @@ -27,9 +33,12 @@ def from_thrift_module(service, thrift_module, thrift_class_name=None): class ThriftRequestMaker(object): - def __init__(self, service, thrift_module, thrift_class_name=None): + def __init__(self, service, thrift_module, + hostport=None, thrift_class_name=None): + self.service = service self.thrift_module = thrift_module + self.hostport = hostport if thrift_class_name is not None: self.thrift_class_name = thrift_class_name @@ -46,7 +55,8 @@ def _make_request(self, method_name, args, kwargs): service=self.service, endpoint=endpoint, result_type=result_type, - call_args=call_args + call_args=call_args, + hostport=self.hostport ) return request @@ -90,11 +100,14 @@ def _get_call_args(self, method_name, args, kwargs): class ThriftRequest(object): - def __init__(self, service, endpoint, result_type, call_args): + def __init__(self, service, endpoint, result_type, + call_args, hostport=None): + self.service = service self.endpoint = endpoint self.result_type = result_type self.call_args = call_args + self.hostport = hostport def _create_methods(thrift_module): diff --git a/tests/schemes/test_json.py b/tests/schemes/test_json.py index de7c7de3..df1c7823 100644 --- a/tests/schemes/test_json.py +++ b/tests/schemes/test_json.py @@ -42,10 +42,11 @@ def endpoint(request, response, proxy): tchannel = TChannel(name='client') resp = yield tchannel.json( - service=server.hostport, + service='server', endpoint='endpoint', headers={'req': 'headers'}, body={'req': 'body'}, + hostport=server.hostport, ) # verify response diff --git a/tests/schemes/test_raw.py b/tests/schemes/test_raw.py index 7f70d1fe..1a696274 100644 --- a/tests/schemes/test_raw.py +++ b/tests/schemes/test_raw.py @@ -38,10 +38,11 @@ def endpoint(request, response, proxy): tchannel = TChannel(name='client') resp = yield tchannel.raw( - service=server.hostport, + service='server', endpoint='endpoint', headers='raw req headers', - body='raw req body' + body='raw req body', + hostport=server.hostport, ) # verify response diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 68d7acf2..af520148 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -47,8 +47,9 @@ def testStruct(request, response, proxy): tchannel = TChannel(name='client') service = from_thrift_module( - service=server.hostport, - thrift_module=ThriftTest + service='service', + thrift_module=ThriftTest, + hostport=server.hostport ) resp = yield tchannel.thrift( diff --git a/tests/test_tchannel.py b/tests/test_tchannel.py index a6cddfac..426fb6c2 100644 --- a/tests/test_tchannel.py +++ b/tests/test_tchannel.py @@ -50,10 +50,11 @@ def endpoint(request, response, proxy): resp = yield tchannel.call( scheme=schemes.RAW, - service=server.hostport, + service='server', arg1='endpoint', arg2='raw req headers', - arg3='raw req body' + arg3='raw req body', + hostport=server.hostport, ) # verify response From 3af9e26884e9becb69b2671b7e420768340e7b03 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 13:59:01 -0700 Subject: [PATCH 08/51] fix headers being required for thrift call to work --- tchannel/schemes/thrift.py | 3 ++ tests/schemes/test_thrift.py | 86 ++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index 76438cf3..5d15e09a 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -19,6 +19,9 @@ def __init__(self, tchannel): def __call__(self, request=None, headers=None, timeout=None, retry_on=None, retry_limit=None): + if headers is None: + headers = {} + # serialize headers = serializer.serialize_headers(headers=headers) body = serializer.serialize_body(call_args=request.call_args) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index af520148..8d955f41 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -20,6 +20,49 @@ def test_call_should_get_response(): server = DeprecatedTChannel(name='server') + @server.register(ThriftTest) + def testStruct(request, response, proxy): + + assert request.args.thing.string_thing == 'req string' + + return ThriftTest.Xtruct( + string_thing="resp string" + ) + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='service', + thrift_module=ThriftTest, + hostport=server.hostport + ) + + resp = yield tchannel.thrift( + request=service.testStruct(ThriftTest.Xtruct("req string")), + ) + + # verify response + assert isinstance(resp, response.Response) + assert resp.body == ThriftTest.Xtruct("resp string") + + # verify response transport headers + assert isinstance(resp.transport, response.ResponseTransportHeaders) + assert resp.transport.scheme == schemes.THRIFT + assert resp.transport.failure_domain is None + + +@pytest.mark.gen_test +@pytest.mark.call +def test_call_should_get_response_with_application_headers(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + @server.register(ThriftTest) def testStruct(request, response, proxy): @@ -66,3 +109,46 @@ def testStruct(request, response, proxy): assert isinstance(resp.transport, response.ResponseTransportHeaders) assert resp.transport.scheme == schemes.THRIFT assert resp.transport.failure_domain is None + + +@pytest.mark.gen_test +@pytest.mark.callz +def test_call_should_get_application_exception(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testMultiException(request, response, proxy): + + if request.args.arg0 == 'Xception': + raise ThriftTest.Xception( + errorCode=1001, + message='This is an Xception', + ) + elif request.args.arg0 == 'Xception2': + raise ThriftTest.Xception2( + errorCode=2002, + message='This is an Xception2', + ) + + return ThriftTest.Xtruct(string_thing=request.args.arg1) + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='service', + thrift_module=ThriftTest, + hostport=server.hostport + ) + + with pytest.raises(ThriftTest.Xception): + yield tchannel.thrift( + request=service.testMultiException(arg0='Xception', arg1='thingy') + ) + + with pytest.raises(ThriftTest.Xception2): + yield tchannel.thrift( + request=service.testMultiException(arg0='Xception2', arg1='thingy') + ) From 13b27afd74741cc97ef7c876c6914153e6f1d035 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 14:00:19 -0700 Subject: [PATCH 09/51] should still assert headers is blank --- tests/schemes/test_thrift.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 8d955f41..7b956dde 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -47,6 +47,7 @@ def testStruct(request, response, proxy): # verify response assert isinstance(resp, response.Response) + assert resp.headers == {} assert resp.body == ThriftTest.Xtruct("resp string") # verify response transport headers From 8014d0e094527baf02cd90961f3666e85aa8f17c Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 14:01:29 -0700 Subject: [PATCH 10/51] use shorthand tchannel.thrift call where possible --- tests/schemes/test_thrift.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 7b956dde..8934bd45 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -42,7 +42,7 @@ def testStruct(request, response, proxy): ) resp = yield tchannel.thrift( - request=service.testStruct(ThriftTest.Xtruct("req string")), + service.testStruct(ThriftTest.Xtruct("req string")) ) # verify response @@ -97,7 +97,7 @@ def testStruct(request, response, proxy): ) resp = yield tchannel.thrift( - request=service.testStruct(ThriftTest.Xtruct("req string")), + service.testStruct(ThriftTest.Xtruct("req string")), headers={'req': 'header'}, ) @@ -146,10 +146,10 @@ def testMultiException(request, response, proxy): with pytest.raises(ThriftTest.Xception): yield tchannel.thrift( - request=service.testMultiException(arg0='Xception', arg1='thingy') + service.testMultiException(arg0='Xception', arg1='thingy') ) with pytest.raises(ThriftTest.Xception2): yield tchannel.thrift( - request=service.testMultiException(arg0='Xception2', arg1='thingy') + service.testMultiException(arg0='Xception2', arg1='thingy') ) From 90092bbe53e8b051afe0182590eac9aef0391c78 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 14:04:27 -0700 Subject: [PATCH 11/51] remove extraneous funcs --- tchannel/schemes/json.py | 6 ------ tchannel/schemes/raw.py | 6 ------ tchannel/schemes/thrift.py | 6 ------ 3 files changed, 18 deletions(-) diff --git a/tchannel/schemes/json.py b/tchannel/schemes/json.py index 265274d5..7947876c 100644 --- a/tchannel/schemes/json.py +++ b/tchannel/schemes/json.py @@ -50,9 +50,3 @@ def __call__(self, service, endpoint, body=None, headers=None, response.body = json.loads(response.body) raise gen.Return(response) - - def stream(self): - pass - - def register(self): - pass diff --git a/tchannel/schemes/raw.py b/tchannel/schemes/raw.py index 019eb3f6..e8bb13ac 100644 --- a/tchannel/schemes/raw.py +++ b/tchannel/schemes/raw.py @@ -27,9 +27,3 @@ def __call__(self, service, endpoint, body=None, headers=None, retry_limit=retry_limit, hostport=hostport, ) - - def stream(self): - pass - - def register(self): - pass diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index 5d15e09a..b7ccf515 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -49,9 +49,3 @@ def __call__(self, request=None, headers=None, timeout=None, response.body = body.success raise gen.Return(response) - - def stream(self): - pass - - def register(self): - pass From a7bf55dc8833cf3f44f4eb87005a196a1feb2e7e Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 15:01:23 -0700 Subject: [PATCH 12/51] add thrift application exception support --- tchannel/schemes/thrift.py | 12 ++++++++++-- tests/schemes/test_thrift.py | 7 ++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index b7ccf515..101a89ae 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -38,7 +38,8 @@ def __call__(self, request=None, headers=None, timeout=None, hostport=request.hostport ) - # deserialize + # deserialize... + response.headers = serializer.deserialize_headers( headers=response.headers ) @@ -46,6 +47,13 @@ def __call__(self, request=None, headers=None, timeout=None, body=response.body, result_type=request.result_type ) - response.body = body.success + # raise application exception, if present + for exc_spec in request.result_type.thrift_spec[1:]: + exc = getattr(body, exc_spec[2]) + if exc is not None: + raise exc + + # success + response.body = body.success raise gen.Return(response) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 8934bd45..2f5a000e 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -113,7 +113,7 @@ def testStruct(request, response, proxy): @pytest.mark.gen_test -@pytest.mark.callz +@pytest.mark.call def test_call_should_get_application_exception(): # Given this test server: @@ -130,12 +130,13 @@ def testMultiException(request, response, proxy): ) elif request.args.arg0 == 'Xception2': raise ThriftTest.Xception2( - errorCode=2002, - message='This is an Xception2', + errorCode=2002 ) return ThriftTest.Xtruct(string_thing=request.args.arg1) + server.listen() + tchannel = TChannel(name='client') service = from_thrift_module( From 094f4a6b62469aa8055032bc98bb9822b1f4bc9a Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 15:10:02 -0700 Subject: [PATCH 13/51] test protocol error --- tests/schemes/test_thrift.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 2f5a000e..5f3cb137 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -10,6 +10,7 @@ ) from tchannel.tornado import TChannel as DeprecatedTChannel from tests.data.generated.ThriftTest import ThriftTest +from tchannel.errors import ProtocolError @pytest.mark.gen_test @@ -137,6 +138,8 @@ def testMultiException(request, response, proxy): server.listen() + # Make a call: + tchannel = TChannel(name='client') service = from_thrift_module( @@ -154,3 +157,33 @@ def testMultiException(request, response, proxy): yield tchannel.thrift( service.testMultiException(arg0='Xception2', arg1='thingy') ) + + +@pytest.mark.gen_test +@pytest.mark.call +def test_call_unexpected_error_should_result_in_protocol_error(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testMultiException(request, response, proxy): + raise Exception('well, this is unfortunate') + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='service', + thrift_module=ThriftTest, + hostport=server.hostport + ) + + with pytest.raises(ProtocolError): + yield tchannel.thrift( + service.testMultiException(arg0='Xception', arg1='thingy') + ) From f660bd1edc900a123a46bf7e260716488d4986ee Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 16:24:48 -0700 Subject: [PATCH 14/51] raw examples --- examples/raw/client.py | 19 ++++++++++--------- examples/raw/server.py | 22 ++++++++++++++-------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/examples/raw/client.py b/examples/raw/client.py index c1d9c8f7..372771f2 100644 --- a/examples/raw/client.py +++ b/examples/raw/client.py @@ -3,26 +3,27 @@ from tchannel import TChannel -client = TChannel('raw-client') +tchannel = TChannel('raw-client') @gen.coroutine def make_request(): - resp = yield client.raw( - service='10.32.160.131:52250', - endpoint='health' + resp = yield tchannel.raw( + service='raw-server', + endpoint='endpoint', + body='req body', + headers='req headers', + hostport='127.0.0.1:54495', ) raise gen.Return(resp) -io_loop = ioloop.IOLoop.current() +resp = ioloop.IOLoop.current().run_sync(make_request) -resp = io_loop.run_sync(make_request) +assert resp.headers == 'resp header' +assert resp.body == 'resp body' - -# TODO impl __repr__ print resp.body print resp.headers -print resp.transport diff --git a/examples/raw/server.py b/examples/raw/server.py index 66f54f59..3670ed6f 100644 --- a/examples/raw/server.py +++ b/examples/raw/server.py @@ -1,21 +1,27 @@ -from tornado.ioloop import IOLoop +from tornado import ioloop, gen from tchannel.tornado import TChannel -app = TChannel('raw-server') +app = TChannel('raw-server', hostport='127.0.0.1:54495') -@app.register('health') -def health(request, response, tchannel): +@app.register('endpoint') +@gen.coroutine +def endpoint(request, response, proxy): - # TODO should be able to return body no matter the scheme - response.write_body('OK') + header = yield request.get_header() + body = yield request.get_body() + + assert header == 'req headers' + assert body == 'req body' + + response.write_header('resp header') + response.write_body('resp body') app.listen() print app.hostport -io_loop = IOLoop.current() -io_loop.start() +ioloop.IOLoop.current().start() From 0516ca65805028e99d5bd7bee504ca4c6339a7ae Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 16:31:57 -0700 Subject: [PATCH 15/51] json examples --- examples/json/client.py | 37 +++++++++++++++++++++++++++++++++++++ examples/json/server.py | 34 +++++++++++++++++----------------- examples/raw/server.py | 2 +- exapmles/json/client.py | 35 ----------------------------------- 4 files changed, 55 insertions(+), 53 deletions(-) create mode 100644 examples/json/client.py delete mode 100644 exapmles/json/client.py diff --git a/examples/json/client.py b/examples/json/client.py new file mode 100644 index 00000000..ef6eb8d4 --- /dev/null +++ b/examples/json/client.py @@ -0,0 +1,37 @@ +from tornado import gen, ioloop + +from tchannel import TChannel + + +tchannel = TChannel('json-client') + + +@gen.coroutine +def make_request(): + + resp = yield tchannel.json( + service='json-server', + endpoint='endpoint', + body={ + 'req': 'body' + }, + headers={ + 'req': 'header' + }, + hostport='127.0.0.1:54496', + ) + + raise gen.Return(resp) + + +resp = ioloop.IOLoop.current().run_sync(make_request) + +assert resp.headers == { + 'resp': 'header', +} +assert resp.body == { + 'resp': 'body', +} + +print resp.body +print resp.headers diff --git a/examples/json/server.py b/examples/json/server.py index d9ad8f1b..bf95c42c 100644 --- a/examples/json/server.py +++ b/examples/json/server.py @@ -1,30 +1,31 @@ -from tornado import gen -from tornado.ioloop import IOLoop +from tornado import gen, ioloop -from tchannel.tornado import TChannel from tchannel import schemes +from tchannel.tornado import TChannel -app = TChannel('json-server') +app = TChannel('json-server', hostport='127.0.0.1:54496') -@app.register('health', schemes.JSON) +@app.register('endpoint', schemes.JSON) @gen.coroutine -def health(request, response, tchannel): +def health(request, response, proxy): - # TODO in thrift you dont have to yield... + header = yield request.get_header() body = yield request.get_body() - # TODO should be get_headers() - headers = yield request.get_header() + assert header == { + 'req': 'header', + } + assert body == { + 'req': 'body', + } - # TODO should be write_headers() - response.write_header(headers) - - # TODO should be able to return body no matter the scheme + response.write_header({ + 'resp': 'header', + }) response.write_body({ - 'health': 'OK', - 'body': body, + 'resp': 'body', }) @@ -32,5 +33,4 @@ def health(request, response, tchannel): print app.hostport -io_loop = IOLoop.current() -io_loop.start() +ioloop.IOLoop.current().start() diff --git a/examples/raw/server.py b/examples/raw/server.py index 3670ed6f..78e7c40c 100644 --- a/examples/raw/server.py +++ b/examples/raw/server.py @@ -1,4 +1,4 @@ -from tornado import ioloop, gen +from tornado import gen, ioloop from tchannel.tornado import TChannel diff --git a/exapmles/json/client.py b/exapmles/json/client.py deleted file mode 100644 index 8261f5e7..00000000 --- a/exapmles/json/client.py +++ /dev/null @@ -1,35 +0,0 @@ -from tornado import gen, ioloop - -from tchannel import TChannel - - -client = TChannel('json-client') -service = '10.32.160.131:52275' - - -@gen.coroutine -def make_request(): - - resp = yield client.json( - service=service, - endpoint='health', - body={ - 'boomboomboom': 'hearyousaywayo' - }, - headers={ - 'bobby': 'twotoes' - } - ) - - raise gen.Return(resp) - - -io_loop = ioloop.IOLoop.current() - -resp = io_loop.run_sync(make_request) - - -# TODO impl __repr__ -print resp.body -print resp.headers -print resp.transport From 6e4ffa9614a2d3c1cddfdc6d581ef5313977a018 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 16:45:04 -0700 Subject: [PATCH 16/51] thrift examples --- examples/json/server.py | 2 +- examples/thrift/client.py | 28 ++++++++++++++-------------- examples/thrift/server.py | 16 ++++++++-------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/examples/json/server.py b/examples/json/server.py index bf95c42c..caac19fe 100644 --- a/examples/json/server.py +++ b/examples/json/server.py @@ -9,7 +9,7 @@ @app.register('endpoint', schemes.JSON) @gen.coroutine -def health(request, response, proxy): +def endpoint(request, response, proxy): header = yield request.get_header() body = yield request.get_body() diff --git a/examples/thrift/client.py b/examples/thrift/client.py index f2a85804..a6f00b56 100644 --- a/examples/thrift/client.py +++ b/examples/thrift/client.py @@ -7,32 +7,32 @@ tchannel = TChannel('thrift-client') -service = '10.32.160.131:52294' -thrift_client = from_thrift_module(service, ThriftTest) +service = from_thrift_module( + service='thrift-server', + thrift_module=ThriftTest, + hostport='127.0.0.1:54497' +) @gen.coroutine def make_request(): resp = yield tchannel.thrift( - request=thrift_client.testString(thing="holler"), - - # TODO bug, you have to have headers :P - headers={'wtf': 'dude'} + request=service.testString(thing="req"), + headers={ + 'req': 'header', + }, ) raise gen.Return(resp) -io_loop = ioloop.IOLoop.current() - -resp = io_loop.run_sync(make_request) +resp = ioloop.IOLoop.current().run_sync(make_request) +assert resp.headers == { + 'resp': 'header', +} +assert resp.body == 'resp' -# TODO impl __repr__ print resp.body - -# TODO wtf tests have headers... print resp.headers - -print resp.transport diff --git a/examples/thrift/server.py b/examples/thrift/server.py index e4807650..1f228870 100644 --- a/examples/thrift/server.py +++ b/examples/thrift/server.py @@ -1,26 +1,26 @@ -from tornado import gen -from tornado.ioloop import IOLoop +from tornado import gen, ioloop from tchannel.tornado import TChannel from tests.data.generated.ThriftTest import ThriftTest -app = TChannel('thrift-server') + +app = TChannel('thrift-server', hostport='127.0.0.1:54497') @app.register(ThriftTest) @gen.coroutine def testString(request, response, tchannel): - # TODO different than other schemes... - response.write_header('hey', 'jane') + assert request.headers == [['req', 'header']] + assert request.args.thing == 'req' - return request.args.thing + response.write_header('resp', 'header') + response.write_result('resp') app.listen() print app.hostport -io_loop = IOLoop.current() -io_loop.start() +ioloop.IOLoop.current().start() From 27679ff4476c7b624fd042741cdbba83a0b3f929 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 17:52:30 -0700 Subject: [PATCH 17/51] examples get ran as part of test suite --- examples/json/client.py | 6 +++-- examples/thrift/client.py | 4 ++- tests/test_examples.py | 55 +++++++++++++++++++++++++++++---------- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/examples/json/client.py b/examples/json/client.py index ef6eb8d4..8e189bef 100644 --- a/examples/json/client.py +++ b/examples/json/client.py @@ -1,3 +1,5 @@ +import json + from tornado import gen, ioloop from tchannel import TChannel @@ -33,5 +35,5 @@ def make_request(): 'resp': 'body', } -print resp.body -print resp.headers +print json.dumps(resp.body) +print json.dumps(resp.headers) diff --git a/examples/thrift/client.py b/examples/thrift/client.py index a6f00b56..d5c06257 100644 --- a/examples/thrift/client.py +++ b/examples/thrift/client.py @@ -1,3 +1,5 @@ +import json + from tornado import gen, ioloop from tchannel import TChannel, from_thrift_module @@ -35,4 +37,4 @@ def make_request(): assert resp.body == 'resp' print resp.body -print resp.headers +print json.dumps(resp.headers) diff --git a/tests/test_examples.py b/tests/test_examples.py index cba88b56..c4ecf0b6 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -19,6 +19,7 @@ # THE SOFTWARE. import contextlib +import json import os import subprocess @@ -62,30 +63,56 @@ def examples_dir(): @pytest.mark.parametrize( - 'example_type', - [ - 'raw_', - 'json_', - 'thrift_examples/', - 'keyvalue/keyvalue/', - 'stream_', - ] + 'scheme, path', + ( + ('raw', 'raw/'), + ('json', 'json/'), + ('thrift', 'thrift/'), + ('thrift', 'keyvalue/keyvalue'), + ) ) -def test_example(examples_dir, example_type): +def test_example(examples_dir, scheme, path): """Smoke test example code to ensure it still runs.""" server_path = os.path.join( examples_dir, - example_type + 'server.py', + path + 'server.py', ) client_path = os.path.join( examples_dir, - example_type + 'client.py', + path + 'client.py', ) with popen(server_path, wait_for_listen=True): with popen(client_path) as client: - assert ( - client.stdout.read() == 'Hello, world!\n' - ), client.stderr.read() + + out = client.stdout.read() + + body, headers = out.split(os.linesep)[:-1] + + if scheme == 'raw': + + assert body == 'resp body' + assert headers == 'resp header' + + elif scheme == 'json': + + body = json.loads(body) + headers = json.loads(headers) + + assert body == { + 'resp': 'body' + } + assert headers == { + 'resp': 'header' + } + + elif scheme == 'thrift': + + headers = json.loads(headers) + + assert body == 'resp' + assert headers == { + 'resp': 'header', + } From e496991ec120f05a730fd7c54308de90bb60c049 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 17:54:33 -0700 Subject: [PATCH 18/51] organize examples --- examples/{ => guide}/keyvalue/keyvalue/__init__.py | 0 examples/{ => guide}/keyvalue/keyvalue/client.py | 0 examples/{ => guide}/keyvalue/keyvalue/server.py | 0 .../{ => guide}/keyvalue/keyvalue/service/KeyValue-remote | 0 .../{ => guide}/keyvalue/keyvalue/service/KeyValue.py | 0 .../{ => guide}/keyvalue/keyvalue/service/__init__.py | 0 .../{ => guide}/keyvalue/keyvalue/service/constants.py | 0 examples/{ => guide}/keyvalue/keyvalue/service/ttypes.py | 0 examples/{ => guide}/keyvalue/setup.py | 0 examples/{ => guide}/keyvalue/thrift/service.thrift | 0 examples/{ => simple}/json/client.py | 0 examples/{ => simple}/json/server.py | 0 examples/{ => simple}/raw/client.py | 0 examples/{ => simple}/raw/server.py | 0 examples/{ => simple}/thrift/client.py | 0 examples/{ => simple}/thrift/server.py | 0 tests/test_examples.py | 8 ++++---- 17 files changed, 4 insertions(+), 4 deletions(-) rename examples/{ => guide}/keyvalue/keyvalue/__init__.py (100%) rename examples/{ => guide}/keyvalue/keyvalue/client.py (100%) rename examples/{ => guide}/keyvalue/keyvalue/server.py (100%) rename examples/{ => guide}/keyvalue/keyvalue/service/KeyValue-remote (100%) rename examples/{ => guide}/keyvalue/keyvalue/service/KeyValue.py (100%) rename examples/{ => guide}/keyvalue/keyvalue/service/__init__.py (100%) rename examples/{ => guide}/keyvalue/keyvalue/service/constants.py (100%) rename examples/{ => guide}/keyvalue/keyvalue/service/ttypes.py (100%) rename examples/{ => guide}/keyvalue/setup.py (100%) rename examples/{ => guide}/keyvalue/thrift/service.thrift (100%) rename examples/{ => simple}/json/client.py (100%) rename examples/{ => simple}/json/server.py (100%) rename examples/{ => simple}/raw/client.py (100%) rename examples/{ => simple}/raw/server.py (100%) rename examples/{ => simple}/thrift/client.py (100%) rename examples/{ => simple}/thrift/server.py (100%) diff --git a/examples/keyvalue/keyvalue/__init__.py b/examples/guide/keyvalue/keyvalue/__init__.py similarity index 100% rename from examples/keyvalue/keyvalue/__init__.py rename to examples/guide/keyvalue/keyvalue/__init__.py diff --git a/examples/keyvalue/keyvalue/client.py b/examples/guide/keyvalue/keyvalue/client.py similarity index 100% rename from examples/keyvalue/keyvalue/client.py rename to examples/guide/keyvalue/keyvalue/client.py diff --git a/examples/keyvalue/keyvalue/server.py b/examples/guide/keyvalue/keyvalue/server.py similarity index 100% rename from examples/keyvalue/keyvalue/server.py rename to examples/guide/keyvalue/keyvalue/server.py diff --git a/examples/keyvalue/keyvalue/service/KeyValue-remote b/examples/guide/keyvalue/keyvalue/service/KeyValue-remote similarity index 100% rename from examples/keyvalue/keyvalue/service/KeyValue-remote rename to examples/guide/keyvalue/keyvalue/service/KeyValue-remote diff --git a/examples/keyvalue/keyvalue/service/KeyValue.py b/examples/guide/keyvalue/keyvalue/service/KeyValue.py similarity index 100% rename from examples/keyvalue/keyvalue/service/KeyValue.py rename to examples/guide/keyvalue/keyvalue/service/KeyValue.py diff --git a/examples/keyvalue/keyvalue/service/__init__.py b/examples/guide/keyvalue/keyvalue/service/__init__.py similarity index 100% rename from examples/keyvalue/keyvalue/service/__init__.py rename to examples/guide/keyvalue/keyvalue/service/__init__.py diff --git a/examples/keyvalue/keyvalue/service/constants.py b/examples/guide/keyvalue/keyvalue/service/constants.py similarity index 100% rename from examples/keyvalue/keyvalue/service/constants.py rename to examples/guide/keyvalue/keyvalue/service/constants.py diff --git a/examples/keyvalue/keyvalue/service/ttypes.py b/examples/guide/keyvalue/keyvalue/service/ttypes.py similarity index 100% rename from examples/keyvalue/keyvalue/service/ttypes.py rename to examples/guide/keyvalue/keyvalue/service/ttypes.py diff --git a/examples/keyvalue/setup.py b/examples/guide/keyvalue/setup.py similarity index 100% rename from examples/keyvalue/setup.py rename to examples/guide/keyvalue/setup.py diff --git a/examples/keyvalue/thrift/service.thrift b/examples/guide/keyvalue/thrift/service.thrift similarity index 100% rename from examples/keyvalue/thrift/service.thrift rename to examples/guide/keyvalue/thrift/service.thrift diff --git a/examples/json/client.py b/examples/simple/json/client.py similarity index 100% rename from examples/json/client.py rename to examples/simple/json/client.py diff --git a/examples/json/server.py b/examples/simple/json/server.py similarity index 100% rename from examples/json/server.py rename to examples/simple/json/server.py diff --git a/examples/raw/client.py b/examples/simple/raw/client.py similarity index 100% rename from examples/raw/client.py rename to examples/simple/raw/client.py diff --git a/examples/raw/server.py b/examples/simple/raw/server.py similarity index 100% rename from examples/raw/server.py rename to examples/simple/raw/server.py diff --git a/examples/thrift/client.py b/examples/simple/thrift/client.py similarity index 100% rename from examples/thrift/client.py rename to examples/simple/thrift/client.py diff --git a/examples/thrift/server.py b/examples/simple/thrift/server.py similarity index 100% rename from examples/thrift/server.py rename to examples/simple/thrift/server.py diff --git a/tests/test_examples.py b/tests/test_examples.py index c4ecf0b6..c217bd43 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -65,10 +65,10 @@ def examples_dir(): @pytest.mark.parametrize( 'scheme, path', ( - ('raw', 'raw/'), - ('json', 'json/'), - ('thrift', 'thrift/'), - ('thrift', 'keyvalue/keyvalue'), + ('raw', 'simple/raw/'), + ('json', 'simple/json/'), + ('thrift', 'simple/thrift/'), + ('thrift', 'guide/keyvalue/keyvalue'), ) ) def test_example(examples_dir, scheme, path): From cce0ac44ef7cfbc4a09409e1da6d6fed9dbfff31 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 18:03:30 -0700 Subject: [PATCH 19/51] cleanup examples --- examples/handlers.py | 57 ----- examples/json_client.py | 59 ----- examples/json_server.py | 51 ----- examples/options.py | 39 ---- examples/raw_client.py | 51 ----- examples/raw_server.py | 49 ---- examples/simple/raw/client.py | 2 +- examples/simple/raw/server.py | 2 +- examples/stream_client.py | 80 ------- examples/stream_server.py | 58 ----- examples/thrift_examples/client.py | 41 ---- examples/thrift_examples/hello.thrift | 3 - .../thrift_examples/hello/HelloService.py | 212 ------------------ examples/thrift_examples/hello/__init__.py | 21 -- examples/thrift_examples/hello/constants.py | 34 --- examples/thrift_examples/hello/ttypes.py | 34 --- examples/thrift_examples/server.py | 42 ---- tests/test_examples.py | 2 +- 18 files changed, 3 insertions(+), 834 deletions(-) delete mode 100644 examples/handlers.py delete mode 100755 examples/json_client.py delete mode 100755 examples/json_server.py delete mode 100644 examples/options.py delete mode 100755 examples/raw_client.py delete mode 100755 examples/raw_server.py delete mode 100755 examples/stream_client.py delete mode 100644 examples/stream_server.py delete mode 100644 examples/thrift_examples/client.py delete mode 100644 examples/thrift_examples/hello.thrift delete mode 100644 examples/thrift_examples/hello/HelloService.py delete mode 100644 examples/thrift_examples/hello/__init__.py delete mode 100644 examples/thrift_examples/hello/constants.py delete mode 100644 examples/thrift_examples/hello/ttypes.py delete mode 100644 examples/thrift_examples/server.py diff --git a/examples/handlers.py b/examples/handlers.py deleted file mode 100644 index 6b22b761..00000000 --- a/examples/handlers.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -from __future__ import absolute_import - -import random - -import tornado.gen - - -@tornado.gen.coroutine -def say_hi(request, response, proxy): - yield response.write_body("Hello, world!") - - -@tornado.gen.coroutine -def echo(request, response, proxy): - # stream args right back to request side - response.set_header_s(request.get_header_s()) - response.set_body_s(request.get_body_s()) - - -@tornado.gen.coroutine -def slow(request, response, proxy): - yield tornado.gen.sleep(random.random()) - yield response.write_body("done") - response.flush() - - -def register_example_endpoints(tchannel): - tchannel.register(endpoint="hi", scheme="raw", handler=say_hi) - tchannel.register(endpoint="echo", scheme="raw", handler=echo) - tchannel.register(endpoint="slow", scheme="raw", handler=slow) - - @tchannel.register("bye", scheme="raw") - def say_bye(request, response, proxy): - print (yield request.get_header()) - print (yield request.get_body()) - - response.write_body("world") diff --git a/examples/json_client.py b/examples/json_client.py deleted file mode 100755 index a48065fa..00000000 --- a/examples/json_client.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -from __future__ import absolute_import - -import tornado.ioloop -import tornado.iostream - -from options import get_args -from tchannel.tornado import TChannel -from tchannel.tornado.broker import ArgSchemeBroker -from tchannel.scheme import JsonArgScheme - - -@tornado.gen.coroutine -def main(): - - args = get_args() - - tchannel = TChannel(name='json-client') - - # TODO: Make this API friendly. - request = tchannel.request( - hostport='%s:%s' % (args.host, args.port), - ) - - response = yield ArgSchemeBroker(JsonArgScheme()).send( - request, - 'hi-json', - None, - None, - ) - - body = yield response.get_body() - - print body['hi'] - - -if __name__ == '__main__': - tornado.ioloop.IOLoop.instance().run_sync(main) diff --git a/examples/json_server.py b/examples/json_server.py deleted file mode 100755 index 4ee0e150..00000000 --- a/examples/json_server.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -from __future__ import absolute_import - -import tornado.ioloop - -from handlers import register_example_endpoints -from options import get_args -from tchannel.tornado import TChannel - - -def main(): - args = get_args() - - app = TChannel( - name='json-server', - hostport='%s:%d' % (args.host, args.port), - ) - - register_example_endpoints(app) - - def say_hi_json(request, response, proxy): - response.write_body({'hi': 'Hello, world!'}) - - app.register(endpoint="hi-json", scheme="json", handler=say_hi_json) - - app.listen() - - tornado.ioloop.IOLoop.instance().start() - - -if __name__ == '__main__': # pragma: no cover - main() diff --git a/examples/options.py b/examples/options.py deleted file mode 100644 index daa362c6..00000000 --- a/examples/options.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -import argparse - - -def get_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--port", - dest="port", default=8888, type=int, - ) - parser.add_argument( - "--host", - dest="host", default="localhost" - ) - return parser - - -def get_args(): - parser = get_parser() - return parser.parse_args() diff --git a/examples/raw_client.py b/examples/raw_client.py deleted file mode 100755 index af2024de..00000000 --- a/examples/raw_client.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -from __future__ import absolute_import - -import tornado.ioloop -import tornado.iostream - -from options import get_args -from tchannel.tornado import TChannel - - -@tornado.gen.coroutine -def main(): - - args = get_args() - - tchannel = TChannel(name='raw-client') - - request = tchannel.request( - hostport='%s:%s' % (args.host, args.port), - ) - - response = yield request.send('hi', None, None) - - body = yield response.get_body() - - print body - - -if __name__ == '__main__': - tornado.ioloop.IOLoop.instance().run_sync(main) diff --git a/examples/raw_server.py b/examples/raw_server.py deleted file mode 100755 index 8efd7a71..00000000 --- a/examples/raw_server.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -import sys - -import tornado.ioloop - -from handlers import register_example_endpoints -from options import get_args -from tchannel.tornado import TChannel - - -def main(): - args = get_args() - - app = TChannel( - name='raw-server', - hostport='%s:%d' % (args.host, args.port), - ) - - register_example_endpoints(app) - - app.listen() - - print("listening on %s" % app.hostport) - sys.stdout.flush() - - tornado.ioloop.IOLoop.instance().start() - - -if __name__ == '__main__': # pragma: no cover - main() diff --git a/examples/simple/raw/client.py b/examples/simple/raw/client.py index 372771f2..5cdb1742 100644 --- a/examples/simple/raw/client.py +++ b/examples/simple/raw/client.py @@ -22,7 +22,7 @@ def make_request(): resp = ioloop.IOLoop.current().run_sync(make_request) -assert resp.headers == 'resp header' +assert resp.headers == 'resp headers' assert resp.body == 'resp body' print resp.body diff --git a/examples/simple/raw/server.py b/examples/simple/raw/server.py index 78e7c40c..59de93a6 100644 --- a/examples/simple/raw/server.py +++ b/examples/simple/raw/server.py @@ -16,7 +16,7 @@ def endpoint(request, response, proxy): assert header == 'req headers' assert body == 'req body' - response.write_header('resp header') + response.write_header('resp headers') response.write_body('resp body') diff --git a/examples/stream_client.py b/examples/stream_client.py deleted file mode 100755 index 96481780..00000000 --- a/examples/stream_client.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -import os -import sys - -import tornado -import tornado.ioloop - -from options import get_parser -from tchannel.tornado import TChannel -from tchannel.tornado.stream import PipeStream - - -@tornado.gen.coroutine -def send_stream(arg1, arg2, arg3, host): - tchannel = TChannel( - name='stream-client', - ) - - response = yield tchannel.request(host).send( - arg1, - arg2, - arg3, - ) - - # Call get_body() to wait for the call to conclude; use - # get_body_s to read the stream as it comes. - body = yield response.get_body() - print body - - -def main(): - parser = get_parser() - parser.add_argument( - "--file", - dest="filename" - ) - args = parser.parse_args() - - hostport = "%s:%s" % (args.host, args.port) - - arg1 = 'hi-stream' - arg2 = None - arg3 = None - - ioloop = tornado.ioloop.IOLoop.current() - - if args.filename == "stdin": - arg3 = PipeStream(sys.stdin.fileno()) - send_stream(arg1, arg2, arg3, hostport) - return ioloop.start() - elif args.filename: - f = os.open(args.filename, os.O_RDONLY) - arg3 = PipeStream(f) - else: - arg3 = 'foo' - - ioloop.run_sync(lambda: send_stream(arg1, arg2, arg3, hostport)) - - -if __name__ == '__main__': # pragma: no cover - main() diff --git a/examples/stream_server.py b/examples/stream_server.py deleted file mode 100644 index c9bc31bc..00000000 --- a/examples/stream_server.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -from __future__ import absolute_import - -import tornado.ioloop - -from handlers import register_example_endpoints -from options import get_args -from tchannel.tornado import TChannel -from tchannel.tornado.stream import InMemStream - - -def main(): - args = get_args() - - app = TChannel( - name='stream-server', - hostport='%s:%d' % (args.host, args.port), - ) - - register_example_endpoints(app) - - @tornado.gen.coroutine - def say_hi_stream(request, response, proxy): - out_stream = InMemStream() - response.set_body_s(out_stream) - - # TODO: Need to be able to flush without closing the stream. - for character in 'Hello, world!': - yield out_stream.write(character) - - app.register(endpoint="hi-stream", handler=say_hi_stream) - - app.listen() - - tornado.ioloop.IOLoop.instance().start() - - -if __name__ == '__main__': # pragma: no cover - main() diff --git a/examples/thrift_examples/client.py b/examples/thrift_examples/client.py deleted file mode 100644 index f9c0149f..00000000 --- a/examples/thrift_examples/client.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -from tornado import gen -from tornado import ioloop - -from hello import HelloService -from tchannel.thrift import client_for -from tchannel.tornado import TChannel - - -@gen.coroutine -def run(): - app = TChannel(name='thrift-client') - - client = client_for('hello', HelloService)(app, 'localhost:8888') - - response = yield client.hello("world") - - print response - - -if __name__ == '__main__': # pragma: no cover - ioloop.IOLoop.current().run_sync(run) diff --git a/examples/thrift_examples/hello.thrift b/examples/thrift_examples/hello.thrift deleted file mode 100644 index 649800d6..00000000 --- a/examples/thrift_examples/hello.thrift +++ /dev/null @@ -1,3 +0,0 @@ -service HelloService { - string hello(1: string name) -} diff --git a/examples/thrift_examples/hello/HelloService.py b/examples/thrift_examples/hello/HelloService.py deleted file mode 100644 index 09ed63fa..00000000 --- a/examples/thrift_examples/hello/HelloService.py +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -# -# Autogenerated by Thrift Compiler (0.9.2) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py:tornado,dynamic,utf8strings -# - -from thrift.protocol.TBase import TBase -from thrift.protocol.TBase import TExceptionBase -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TMessageType -from thrift.Thrift import TProcessor -from thrift.Thrift import TType -from thrift.transport import TTransport -from tornado import concurrent -from tornado import gen - -from ttypes import * - - -class Iface(object): - def hello(self, name): - """ - Parameters: - - name - """ - pass - - -class Client(Iface): - def __init__(self, transport, iprot_factory, oprot_factory=None): - self._transport = transport - self._iprot_factory = iprot_factory - self._oprot_factory = (oprot_factory if oprot_factory is not None - else iprot_factory) - self._seqid = 0 - self._reqs = {} - self._transport.io_loop.spawn_callback(self._start_receiving) - - @gen.engine - def _start_receiving(self): - while True: - try: - frame = yield self._transport.readFrame() - except TTransport.TTransportException as e: - for future in self._reqs.itervalues(): - future.set_exception(e) - self._reqs = {} - return - tr = TTransport.TMemoryBuffer(frame) - iprot = self._iprot_factory.getProtocol(tr) - (fname, mtype, rseqid) = iprot.readMessageBegin() - future = self._reqs.pop(rseqid, None) - if not future: - # future has already been discarded - continue - method = getattr(self, 'recv_' + fname) - try: - result = method(iprot, mtype, rseqid) - except Exception as e: - future.set_exception(e) - else: - future.set_result(result) - - def hello(self, name): - """ - Parameters: - - name - """ - self._seqid += 1 - future = self._reqs[self._seqid] = concurrent.Future() - self.send_hello(name) - return future - - def send_hello(self, name): - oprot = self._oprot_factory.getProtocol(self._transport) - oprot.writeMessageBegin('hello', TMessageType.CALL, self._seqid) - args = hello_args() - args.name = name - args.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def recv_hello(self, iprot, mtype, rseqid): - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = hello_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - raise TApplicationException(TApplicationException.MISSING_RESULT, "hello failed: unknown result"); - - -class Processor(Iface, TProcessor): - def __init__(self, handler): - self._handler = handler - self._processMap = {} - self._processMap["hello"] = Processor.process_hello - - def process(self, iprot, oprot): - (name, type, seqid) = iprot.readMessageBegin() - if name not in self._processMap: - iprot.skip(TType.STRUCT) - iprot.readMessageEnd() - x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) - oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) - x.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - return - else: - return self._processMap[name](self, seqid, iprot, oprot) - - @gen.coroutine - def process_hello(self, seqid, iprot, oprot): - args = hello_args() - args.read(iprot) - iprot.readMessageEnd() - result = hello_result() - result.success = yield gen.maybe_future(self._handler.hello(args.name)) - oprot.writeMessageBegin("hello", TMessageType.REPLY, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - -# HELPER FUNCTIONS AND STRUCTURES - -class hello_args(TBase): - """ - Attributes: - - name - """ - - thrift_spec = ( - None, # 0 - (1, TType.STRING, 'name', None, None, ), # 1 - ) - - def __init__(self, name=None,): - self.name = name - - def __hash__(self): - value = 17 - value = (value * 31) ^ hash(self.name) - return value - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.iteritems()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - -class hello_result(TBase): - """ - Attributes: - - success - """ - - thrift_spec = ( - (0, TType.STRING, 'success', None, None, ), # 0 - ) - - def __init__(self, success=None,): - self.success = success - - def __hash__(self): - value = 17 - value = (value * 31) ^ hash(self.success) - return value - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.iteritems()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) diff --git a/examples/thrift_examples/hello/__init__.py b/examples/thrift_examples/hello/__init__.py deleted file mode 100644 index f5554aa6..00000000 --- a/examples/thrift_examples/hello/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -__all__ = ['ttypes', 'constants', 'HelloService'] diff --git a/examples/thrift_examples/hello/constants.py b/examples/thrift_examples/hello/constants.py deleted file mode 100644 index 3d4cf85c..00000000 --- a/examples/thrift_examples/hello/constants.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -# -# Autogenerated by Thrift Compiler (0.9.2) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py:tornado,dynamic,utf8strings -# - -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TMessageType -from thrift.Thrift import TType - -from ttypes import * diff --git a/examples/thrift_examples/hello/ttypes.py b/examples/thrift_examples/hello/ttypes.py deleted file mode 100644 index 61e947ed..00000000 --- a/examples/thrift_examples/hello/ttypes.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -# -# Autogenerated by Thrift Compiler (0.9.2) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py:tornado,dynamic,utf8strings -# - -from thrift.protocol.TBase import TBase -from thrift.protocol.TBase import TExceptionBase -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TMessageType -from thrift.Thrift import TType diff --git a/examples/thrift_examples/server.py b/examples/thrift_examples/server.py deleted file mode 100644 index f7c3b272..00000000 --- a/examples/thrift_examples/server.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2015 Uber Technologies, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -from tornado import ioloop - -from hello import HelloService -from tchannel.tornado import TChannel - -app = TChannel('thrift-server', 'localhost:8888') - - -@app.register(HelloService) -def hello(request, response, tchannel): - name = request.args.name - print "Hello, %s!" % name - return "Hello, %s!" % name - - -def run(): - app.listen() - ioloop.IOLoop.current().start() - - -if __name__ == '__main__': - run() diff --git a/tests/test_examples.py b/tests/test_examples.py index c217bd43..c42c8de1 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -94,7 +94,7 @@ def test_example(examples_dir, scheme, path): if scheme == 'raw': assert body == 'resp body' - assert headers == 'resp header' + assert headers == 'resp headers' elif scheme == 'json': From b6fb2eeb3800531e16446bf3089eb77b47de4a89 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 18:20:20 -0700 Subject: [PATCH 20/51] fix guide test --- tests/test_examples.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index c42c8de1..5dbe38bc 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -68,7 +68,7 @@ def examples_dir(): ('raw', 'simple/raw/'), ('json', 'simple/json/'), ('thrift', 'simple/thrift/'), - ('thrift', 'guide/keyvalue/keyvalue'), + ('guide', 'guide/keyvalue/keyvalue/'), ) ) def test_example(examples_dir, scheme, path): @@ -89,6 +89,11 @@ def test_example(examples_dir, scheme, path): out = client.stdout.read() + # TODO the guide test should be the same as others + if scheme == 'guide': + assert out == 'Hello, world!\n' + return + body, headers = out.split(os.linesep)[:-1] if scheme == 'raw': From a673d79191f9230ba8a456b259150a9d8ac2226b Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 18:28:02 -0700 Subject: [PATCH 21/51] assert exception meta --- tests/schemes/test_thrift.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 5f3cb137..85866208 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -148,15 +148,17 @@ def testMultiException(request, response, proxy): hostport=server.hostport ) - with pytest.raises(ThriftTest.Xception): + with pytest.raises(ThriftTest.Xception) as e: yield tchannel.thrift( service.testMultiException(arg0='Xception', arg1='thingy') ) + assert e.value.errorCode == 1001 - with pytest.raises(ThriftTest.Xception2): + with pytest.raises(ThriftTest.Xception2) as e: yield tchannel.thrift( service.testMultiException(arg0='Xception2', arg1='thingy') ) + assert e.value.errorCode == 2002 @pytest.mark.gen_test From b32fc9dd5614a049b2dd1bafe377cc16aeeeefb0 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 18:34:39 -0700 Subject: [PATCH 22/51] assert full exeption meta --- tests/schemes/test_thrift.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 85866208..dea9ba14 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -153,6 +153,7 @@ def testMultiException(request, response, proxy): service.testMultiException(arg0='Xception', arg1='thingy') ) assert e.value.errorCode == 1001 + assert e.value.message == 'This is an Xception' with pytest.raises(ThriftTest.Xception2) as e: yield tchannel.thrift( From 485c3352f720dac7fbcbf72ee9856bab5711b2c4 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 18:43:14 -0700 Subject: [PATCH 23/51] use localhost for examples --- examples/simple/json/client.py | 2 +- examples/simple/json/server.py | 2 +- examples/simple/raw/client.py | 2 +- examples/simple/raw/server.py | 2 +- examples/simple/thrift/client.py | 2 +- examples/simple/thrift/server.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/simple/json/client.py b/examples/simple/json/client.py index 8e189bef..76deb63e 100644 --- a/examples/simple/json/client.py +++ b/examples/simple/json/client.py @@ -20,7 +20,7 @@ def make_request(): headers={ 'req': 'header' }, - hostport='127.0.0.1:54496', + hostport='localhost:54496', ) raise gen.Return(resp) diff --git a/examples/simple/json/server.py b/examples/simple/json/server.py index caac19fe..5336a31f 100644 --- a/examples/simple/json/server.py +++ b/examples/simple/json/server.py @@ -4,7 +4,7 @@ from tchannel.tornado import TChannel -app = TChannel('json-server', hostport='127.0.0.1:54496') +app = TChannel('json-server', hostport='localhost:54496') @app.register('endpoint', schemes.JSON) diff --git a/examples/simple/raw/client.py b/examples/simple/raw/client.py index 5cdb1742..e207db87 100644 --- a/examples/simple/raw/client.py +++ b/examples/simple/raw/client.py @@ -14,7 +14,7 @@ def make_request(): endpoint='endpoint', body='req body', headers='req headers', - hostport='127.0.0.1:54495', + hostport='localhost:54495', ) raise gen.Return(resp) diff --git a/examples/simple/raw/server.py b/examples/simple/raw/server.py index 59de93a6..f79297fd 100644 --- a/examples/simple/raw/server.py +++ b/examples/simple/raw/server.py @@ -3,7 +3,7 @@ from tchannel.tornado import TChannel -app = TChannel('raw-server', hostport='127.0.0.1:54495') +app = TChannel('raw-server', hostport='localhost:54495') @app.register('endpoint') diff --git a/examples/simple/thrift/client.py b/examples/simple/thrift/client.py index d5c06257..0e26ed16 100644 --- a/examples/simple/thrift/client.py +++ b/examples/simple/thrift/client.py @@ -12,7 +12,7 @@ service = from_thrift_module( service='thrift-server', thrift_module=ThriftTest, - hostport='127.0.0.1:54497' + hostport='localhost:54497' ) diff --git a/examples/simple/thrift/server.py b/examples/simple/thrift/server.py index 1f228870..b1e330a7 100644 --- a/examples/simple/thrift/server.py +++ b/examples/simple/thrift/server.py @@ -5,7 +5,7 @@ from tests.data.generated.ThriftTest import ThriftTest -app = TChannel('thrift-server', hostport='127.0.0.1:54497') +app = TChannel('thrift-server', hostport='localhost:54497') @app.register(ThriftTest) From 7aa0ccb03ca46827aeb1436447da02f6683e4ccc Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 19:32:21 -0700 Subject: [PATCH 24/51] using listen_on instead of hostport when instantiating TChannel --- tchannel/tchannel.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tchannel/tchannel.py b/tchannel/tchannel.py index 5b73fbbf..4374d4ac 100644 --- a/tchannel/tchannel.py +++ b/tchannel/tchannel.py @@ -14,13 +14,17 @@ class TChannel(object): - def __init__(self, name, hostport=None, process_name=None, + def __init__(self, name, listen_on=None, process_name=None, known_peers=None, trace=False): # until we move everything here, # lets compose the old tchannel self._dep_tchannel = DeprecatedTChannel( - name, hostport, process_name, known_peers, trace + name=name, + hostport=listen_on, + process_name=process_name, + known_peers=known_peers, + trace=trace ) self.name = name @@ -82,7 +86,3 @@ def call(self, scheme, service, arg1, arg2=None, arg3=None, result = Response(header, body, t) raise gen.Return(result) - - -def _is_hostport(service): - return ':' in service From d51df6b43384c459baf5389113ebaca15effa70e Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Tue, 4 Aug 2015 20:42:35 -0700 Subject: [PATCH 25/51] finish rebase --- setup.cfg | 2 +- tchannel/testing/vcr/patch.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 52960e47..8f892cbc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,4 +3,4 @@ create-wheel = yes [flake8] # Ignore files generated by Thrift -exclude = examples/keyvalue/keyvalue/service/*,tchannel/zipkin/thrift/*,tchannel/testing/vcr/proxy/*,tests/data/generated/ThriftTest/* +exclude = examples/guide/keyvalue/keyvalue/service/*,tchannel/zipkin/thrift/*,tchannel/testing/vcr/proxy/*,tests/data/generated/ThriftTest/* diff --git a/tchannel/testing/vcr/patch.py b/tchannel/testing/vcr/patch.py index 4ece9ce4..299732e8 100644 --- a/tchannel/testing/vcr/patch.py +++ b/tchannel/testing/vcr/patch.py @@ -26,12 +26,12 @@ import contextlib2 from tornado import gen +from tchannel import schemes from tchannel.errors import ProtocolError from tchannel.tornado import TChannel from tchannel.tornado.response import Response from tchannel.tornado.stream import maybe_stream from tchannel.tornado.stream import read_full -from tchannel.transport_header import ArgSchemeType from .proxy import VCRProxy @@ -63,7 +63,7 @@ def __init__( self.vcr_client = vcr_client self.hostport = hostport self.service = service or '' - self.arg_scheme = arg_scheme or ArgSchemeType.DEFAULT + self.arg_scheme = arg_scheme or schemes.DEFAULT # TODO what to do with retry, parent_tracing and score_threshold From fcf845a63e5983a16ec505fdcb87e454c1c483f9 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 11:51:39 -0700 Subject: [PATCH 26/51] adding TODO's --- tchannel/response.py | 8 ++++++++ tchannel/schemes/__init__.py | 2 ++ tchannel/tchannel.py | 2 ++ tchannel/thrift/module.py | 13 +++++++++++++ tchannel/thrift/serializer.py | 10 ++++++++++ tchannel/tornado/broker.py | 4 ++-- 6 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tchannel/response.py b/tchannel/response.py index a3148bac..da31c8df 100644 --- a/tchannel/response.py +++ b/tchannel/response.py @@ -8,6 +8,10 @@ class Response(object): + + # TODO add __slots__ + # TODO implement __repr__ + def __init__(self, headers, body, transport): self.headers = headers self.body = body @@ -15,5 +19,9 @@ def __init__(self, headers, body, transport): class ResponseTransportHeaders(transport.TransportHeaders): + + # TODO add __slots__ + # TODO implement __repr__ + """Response-specific Transport Headers""" pass diff --git a/tchannel/schemes/__init__.py b/tchannel/schemes/__init__.py index 9acc152b..edb3e5fc 100644 --- a/tchannel/schemes/__init__.py +++ b/tchannel/schemes/__init__.py @@ -22,3 +22,5 @@ JsonArgScheme, ThriftArgScheme ) + +# TODO move constants to schemes/glossary and import here diff --git a/tchannel/tchannel.py b/tchannel/tchannel.py index 4374d4ac..f2a31f33 100644 --- a/tchannel/tchannel.py +++ b/tchannel/tchannel.py @@ -55,6 +55,8 @@ def call(self, scheme, service, arg1, arg2=None, arg3=None, if retry_limit is None: retry_limit = retry.DEFAULT_RETRY_LIMIT + # TODO - allow filters/steps for serialization, tracing, etc... + # calls tchannel.tornado.peer.PeerClientOperation.__init__ operation = self._dep_tchannel.request( service=service, diff --git a/tchannel/thrift/module.py b/tchannel/thrift/module.py index 7c7263a9..6806fd4c 100644 --- a/tchannel/thrift/module.py +++ b/tchannel/thrift/module.py @@ -100,6 +100,9 @@ def _get_call_args(self, method_name, args, kwargs): class ThriftRequest(object): + # TODO - add __slots__ + # TODO - implement __repr__ + def __init__(self, service, endpoint, result_type, call_args, hostport=None): @@ -112,6 +115,13 @@ def __init__(self, service, endpoint, result_type, def _create_methods(thrift_module): + # TODO - this method isn't needed, instead, do: + # + # for name in get_service_methods(...): + # method = _create_method(...) + # # ... + # + methods = {} method_names = get_service_methods(thrift_module.Iface) @@ -125,6 +135,9 @@ def _create_methods(thrift_module): def _create_method(method_name): + # TODO - copy over entire signature using @functools.wraps(that_function) + # or wrapt on Iface. + def method(self, *args, **kwargs): # TODO switch to __make_request return self._make_request(method_name, args, kwargs) diff --git a/tchannel/thrift/serializer.py b/tchannel/thrift/serializer.py index 12dcb4dc..f4b427de 100644 --- a/tchannel/thrift/serializer.py +++ b/tchannel/thrift/serializer.py @@ -37,6 +37,16 @@ def deserialize_headers(headers): def serialize_body(call_args): + # TODO - use fastbinary directly + # + # fastbinary.encode_binary( + # call_args, (call_args.__class__, call_args.thrift_spec) + # ) + # fastbinary.decode_binary( + # result, TMemoryBuffer(body), (result_type, result_type.thrift_spec) + # ) + # + trans = TTransport.TMemoryBuffer() proto = TBinaryProtocol.TBinaryProtocolAccelerated(trans) call_args.write(proto) diff --git a/tchannel/tornado/broker.py b/tchannel/tornado/broker.py index 271cf24d..3136c7d0 100644 --- a/tchannel/tornado/broker.py +++ b/tchannel/tornado/broker.py @@ -65,10 +65,10 @@ def handle_call(self, req, resp, proxy): @tornado.gen.coroutine def send(self, - client, # operation? + client, endpoint, header, - body, # NOTE body==call_args + body, protocol_headers=None, traceflag=None, attempt_times=None, From c65ca4bccaf83563cd2870d3b3b49ad3bcadfa1a Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 11:56:22 -0700 Subject: [PATCH 27/51] feedback --- tchannel/thrift/serializer.py | 6 +----- tests/test_tchannel.py | 2 ++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tchannel/thrift/serializer.py b/tchannel/thrift/serializer.py index f4b427de..1bf4fdb2 100644 --- a/tchannel/thrift/serializer.py +++ b/tchannel/thrift/serializer.py @@ -26,11 +26,7 @@ def deserialize_headers(headers): headers = io.BytesIO(headers) headers = _headers_rw.read(headers) - - result = {} - - for h in headers: - result[h[0]] = h[1] + result = dict(headers) return result diff --git a/tests/test_tchannel.py b/tests/test_tchannel.py index 426fb6c2..d1916974 100644 --- a/tests/test_tchannel.py +++ b/tests/test_tchannel.py @@ -9,6 +9,8 @@ from tchannel import response from tchannel.tornado import TChannel as DeprecatedTChannel +# TODO - need integration tests for timeout and retries, use testing.vcr + @pytest.mark.call def test_should_have_default_schemes(): From e5b61f60d84b6f9d23fc37934fa54ccaac0e414b Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 13:01:20 -0700 Subject: [PATCH 28/51] add thrift test for each method in TestThrift.thrift, many unimplemented. --- tests/schemes/test_thrift.py | 146 ++++++++++++++++++++++++++++++++++- 1 file changed, 142 insertions(+), 4 deletions(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index dea9ba14..ed5c0eb7 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -15,7 +15,72 @@ @pytest.mark.gen_test @pytest.mark.call -def test_call_should_get_response(): +def test_void(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testVoid(request, response, proxy): + pass + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift(service.testVoid()) + + assert resp is None + + +@pytest.mark.gen_test +@pytest.mark.call +def test_string(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_byte(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_i32(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_i64(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_double(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_binary(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_struct(): # Given this test server: @@ -59,7 +124,7 @@ def testStruct(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call -def test_call_should_get_response_with_application_headers(): +def test_struct_with_headers(): # Given this test server: @@ -115,7 +180,73 @@ def testStruct(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call -def test_call_should_get_application_exception(): +def test_nest(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_map(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_string_map(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_set(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_list(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_enum(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_type_def(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_map_map(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_insanity(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_multi(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_exception(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_multi_exception(): # Given this test server: @@ -162,6 +293,13 @@ def testMultiException(request, response, proxy): assert e.value.errorCode == 2002 +@pytest.mark.gen_test +@pytest.mark.call +def test_oneway(): + # this is currently unsupported + pass + + @pytest.mark.gen_test @pytest.mark.call def test_call_unexpected_error_should_result_in_protocol_error(): @@ -181,7 +319,7 @@ def testMultiException(request, response, proxy): tchannel = TChannel(name='client') service = from_thrift_module( - service='service', + service='server', thrift_module=ThriftTest, hostport=server.hostport ) From f8bacd09d649c6be74c7fbd8f79f00576a54bfee Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 13:31:07 -0700 Subject: [PATCH 29/51] thrift support void --- tchannel/schemes/thrift.py | 15 +++++++++++---- tests/schemes/test_thrift.py | 3 ++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index 101a89ae..4e133fd5 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -47,13 +47,20 @@ def __call__(self, request=None, headers=None, timeout=None, body=response.body, result_type=request.result_type ) + result_spec = request.result_type.thrift_spec # raise application exception, if present - for exc_spec in request.result_type.thrift_spec[1:]: + for exc_spec in result_spec[1:]: exc = getattr(body, exc_spec[2]) if exc is not None: raise exc - # success - response.body = body.success - raise gen.Return(response) + # success - non-void + if len(result_spec) >= 1: + response.body = body.success + raise gen.Return(response) + + # success - void + else: + response.body = None + raise gen.Return(response) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index ed5c0eb7..eb94e4ff 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -39,7 +39,8 @@ def testVoid(request, response, proxy): resp = yield tchannel.thrift(service.testVoid()) - assert resp is None + assert resp.headers == {} + assert resp.body is None @pytest.mark.gen_test From 34cf8c0725dd47329edefbc0e642cbe34654fb83 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 13:37:35 -0700 Subject: [PATCH 30/51] test call response should contain transport headers --- tests/schemes/test_thrift.py | 49 ++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index eb94e4ff..490fb24d 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -117,11 +117,6 @@ def testStruct(request, response, proxy): assert resp.headers == {} assert resp.body == ThriftTest.Xtruct("resp string") - # verify response transport headers - assert isinstance(resp.transport, response.ResponseTransportHeaders) - assert resp.transport.scheme == schemes.THRIFT - assert resp.transport.failure_domain is None - @pytest.mark.gen_test @pytest.mark.call @@ -173,11 +168,6 @@ def testStruct(request, response, proxy): assert resp.headers == {'resp': 'header'} assert resp.body == ThriftTest.Xtruct("resp string") - # verify response transport headers - assert isinstance(resp.transport, response.ResponseTransportHeaders) - assert resp.transport.scheme == schemes.THRIFT - assert resp.transport.failure_domain is None - @pytest.mark.gen_test @pytest.mark.call @@ -301,6 +291,43 @@ def test_oneway(): pass +@pytest.mark.gen_test +@pytest.mark.call +def test_call_response_should_contain_transport_headers(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testString(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift(service.testString('hi')) + + # verify response + assert isinstance(resp, response.Response) + assert resp.headers == {} + assert resp.body == 'hi' + + # verify response transport headers + assert isinstance(resp.transport, response.ResponseTransportHeaders) + assert resp.transport.scheme == schemes.THRIFT + assert resp.transport.failure_domain is None + + @pytest.mark.gen_test @pytest.mark.call def test_call_unexpected_error_should_result_in_protocol_error(): @@ -322,7 +349,7 @@ def testMultiException(request, response, proxy): service = from_thrift_module( service='server', thrift_module=ThriftTest, - hostport=server.hostport + hostport=server.hostport, ) with pytest.raises(ProtocolError): From 6aef0597d52e6b0431e05e62daea58106256d551 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 14:15:18 -0700 Subject: [PATCH 31/51] thrift exception tests --- tchannel/schemes/thrift.py | 2 +- tests/schemes/test_thrift.py | 62 +++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/tchannel/schemes/thrift.py b/tchannel/schemes/thrift.py index 4e133fd5..f19433af 100644 --- a/tchannel/schemes/thrift.py +++ b/tchannel/schemes/thrift.py @@ -56,7 +56,7 @@ def __call__(self, request=None, headers=None, timeout=None, raise exc # success - non-void - if len(result_spec) >= 1: + if len(result_spec) >= 1 and result_spec[0] is not None: response.body = body.success raise gen.Return(response) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 490fb24d..1495ae48 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -232,7 +232,57 @@ def test_multi(): @pytest.mark.gen_test @pytest.mark.call def test_exception(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testException(request, response, proxy): + + if request.args.arg == 'Xception': + raise ThriftTest.Xception( + errorCode=1001, + message=request.args.arg + ) + elif request.args.arg == 'TException': + # TODO - what to raise here? We dont want dep on Thrift + # so we don't have thrift.TException available to us... + raise Exception() + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='service', + thrift_module=ThriftTest, + hostport=server.hostport + ) + + # case #1 + with pytest.raises(ThriftTest.Xception) as e: + yield tchannel.thrift( + service.testException(arg='Xception') + ) + assert e.value.errorCode == 1001 + assert e.value.message == 'Xception' + + # case #2 + with pytest.raises(ProtocolError): + yield tchannel.thrift( + service.testException(arg='TException') + ) + + # case #3 + resp = yield tchannel.thrift( + service.testException(arg='something else') + ) + assert isinstance(resp, response.Response) + assert resp.headers == {} + assert resp.body is None @pytest.mark.gen_test @@ -270,6 +320,7 @@ def testMultiException(request, response, proxy): hostport=server.hostport ) + # case #1 with pytest.raises(ThriftTest.Xception) as e: yield tchannel.thrift( service.testMultiException(arg0='Xception', arg1='thingy') @@ -277,12 +328,21 @@ def testMultiException(request, response, proxy): assert e.value.errorCode == 1001 assert e.value.message == 'This is an Xception' + # case #2 with pytest.raises(ThriftTest.Xception2) as e: yield tchannel.thrift( service.testMultiException(arg0='Xception2', arg1='thingy') ) assert e.value.errorCode == 2002 + # case #3 + resp = yield tchannel.thrift( + service.testMultiException(arg0='something else', arg1='thingy') + ) + assert isinstance(resp, response.Response) + assert resp.headers == {} + assert resp.body == ThriftTest.Xtruct('thingy') + @pytest.mark.gen_test @pytest.mark.call From 9b448be42fa9505f3a46fc4be4dd3084732084b1 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 14:18:42 -0700 Subject: [PATCH 32/51] thrift test void works with headers --- tests/schemes/test_thrift.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 1495ae48..423f28c5 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -43,6 +43,38 @@ def testVoid(request, response, proxy): assert resp.body is None +@pytest.mark.gen_test +@pytest.mark.call +def test_void_with_headers(): + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testVoid(request, response, proxy): + response.write_header('resp', 'header') + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift(service.testVoid()) + + assert resp.headers == { + 'resp': 'header' + } + assert resp.body is None + + @pytest.mark.gen_test @pytest.mark.call def test_string(): From 8055470980e09abb10e023fcdb18c9abd76d6342 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 14:25:07 -0700 Subject: [PATCH 33/51] test thrift string --- tests/schemes/test_thrift.py | 40 +++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 423f28c5..691ad706 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -78,7 +78,33 @@ def testVoid(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_string(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testString(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift( + service.testString('howdy') + ) + + assert resp.headers == {} + assert resp.body == 'howdy' @pytest.mark.gen_test @@ -383,6 +409,18 @@ def test_oneway(): pass +@pytest.mark.gen_test +@pytest.mark.call +def test_second_service_blah_blah(): + pass + + +@pytest.mark.gen_test +@pytest.mark.call +def test_second_service_second_test_string(): + pass + + @pytest.mark.gen_test @pytest.mark.call def test_call_response_should_contain_transport_headers(): From 9b4395b8df6dea26fc6654e96428aab128e3e7c8 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 15:19:34 -0700 Subject: [PATCH 34/51] thrift test second service --- tchannel/errors.py | 5 ++ tchannel/thrift/client.py | 14 ++++- tests/schemes/test_thrift.py | 106 ++++++++++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 6 deletions(-) diff --git a/tchannel/errors.py b/tchannel/errors.py index 98077a13..62b8e90e 100644 --- a/tchannel/errors.py +++ b/tchannel/errors.py @@ -119,3 +119,8 @@ def __init__(self, code, args): self.code = code self.args = args + + +class OneWayNotSupporedError(TChannelError): + """Raised when oneway Thrift procedure is called.""" + pass diff --git a/tchannel/thrift/client.py b/tchannel/thrift/client.py index e3c2ce26..86527990 100644 --- a/tchannel/thrift/client.py +++ b/tchannel/thrift/client.py @@ -26,6 +26,7 @@ from thrift import Thrift from tornado import gen +from tchannel.errors import OneWayNotSupporedError from tchannel.tornado.broker import ArgSchemeBroker from tchannel.dep.thrift_arg_scheme import DeprecatedThriftArgScheme from .reflection import get_service_methods @@ -119,9 +120,16 @@ def generate_method(service_module, service_name, method_name): assert method_name args_type = getattr(service_module, method_name + '_args') - result_type = getattr(service_module, method_name + '_result') - # TODO result_type is None when the method is oneway. - # We don't support oneway yet. + result_type = getattr(service_module, method_name + '_result', None) + + # oneway not currently supported + # TODO - write test for this + if result_type is None: + def not_supported(self, *args, **kwags): + raise OneWayNotSupporedError( + 'TChannel+Thrift does not currently support oneway procedues' + ) + return not_supported arg_scheme = DeprecatedThriftArgScheme(result_type) result_spec = result_type.thrift_spec diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 691ad706..179af75d 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -3,13 +3,15 @@ ) import pytest +from tornado import gen from tchannel import ( TChannel, from_thrift_module, schemes, response ) from tchannel.tornado import TChannel as DeprecatedTChannel -from tests.data.generated.ThriftTest import ThriftTest +from tchannel.thrift import client_for +from tests.data.generated.ThriftTest import ThriftTest, SecondService from tchannel.errors import ProtocolError @@ -412,13 +414,111 @@ def test_oneway(): @pytest.mark.gen_test @pytest.mark.call def test_second_service_blah_blah(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testString(request, response, proxy): + return request.args.thing + + @server.register(SecondService) + def blahBlah(request, response, proxy): + pass + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport + ) + + second_service = from_thrift_module( + service='server', + thrift_module=SecondService, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift(service.testString('thing')) + + assert isinstance(resp, response.Response) + assert resp.headers == {} + assert resp.body == 'thing' + + resp = yield tchannel.thrift(second_service.blahBlah()) + + assert isinstance(resp, response.Response) + assert resp.headers == {} + assert resp.body is None @pytest.mark.gen_test @pytest.mark.call def test_second_service_second_test_string(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testString(request, response, proxy): + return request.args.thing + + @server.register(SecondService) + @gen.coroutine + def secondtestString(request, response, proxy): + + # TODO - is this really how our server thrift story looks? + ThriftTestService = client_for( + service='server', + service_module=ThriftTest + ) + service = ThriftTestService( + tchannel=proxy, + hostport=server.hostport, + ) + + resp = yield service.testString(request.args.thing) + + response.write_result(resp) + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport + ) + + second_service = from_thrift_module( + service='server', + thrift_module=SecondService, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift(service.testString('thing')) + + assert isinstance(resp, response.Response) + assert resp.headers == {} + assert resp.body == 'thing' + + resp = yield tchannel.thrift( + second_service.secondtestString('second_string') + ) + + assert isinstance(resp, response.Response) + assert resp.headers == {} + assert resp.body == 'second_string' @pytest.mark.gen_test From e24fe941c90e0d8578a656121d00873c1d0d6395 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 15:31:12 -0700 Subject: [PATCH 35/51] thrift test oneway raises exception when called --- tchannel/errors.py | 2 +- tchannel/thrift/client.py | 4 ++-- tchannel/thrift/module.py | 14 ++++++++++++-- tests/schemes/test_thrift.py | 28 ++++++++++++++++++++++++++-- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/tchannel/errors.py b/tchannel/errors.py index 62b8e90e..e6f0441f 100644 --- a/tchannel/errors.py +++ b/tchannel/errors.py @@ -121,6 +121,6 @@ def __init__(self, code, args): self.args = args -class OneWayNotSupporedError(TChannelError): +class OneWayNotSupportedError(TChannelError): """Raised when oneway Thrift procedure is called.""" pass diff --git a/tchannel/thrift/client.py b/tchannel/thrift/client.py index 86527990..dfa9c2c4 100644 --- a/tchannel/thrift/client.py +++ b/tchannel/thrift/client.py @@ -26,7 +26,7 @@ from thrift import Thrift from tornado import gen -from tchannel.errors import OneWayNotSupporedError +from tchannel.errors import OneWayNotSupportedError from tchannel.tornado.broker import ArgSchemeBroker from tchannel.dep.thrift_arg_scheme import DeprecatedThriftArgScheme from .reflection import get_service_methods @@ -126,7 +126,7 @@ def generate_method(service_module, service_name, method_name): # TODO - write test for this if result_type is None: def not_supported(self, *args, **kwags): - raise OneWayNotSupporedError( + raise OneWayNotSupportedError( 'TChannel+Thrift does not currently support oneway procedues' ) return not_supported diff --git a/tchannel/thrift/module.py b/tchannel/thrift/module.py index 6806fd4c..0bf14f8e 100644 --- a/tchannel/thrift/module.py +++ b/tchannel/thrift/module.py @@ -5,6 +5,7 @@ import inspect import types +from tchannel.errors import OneWayNotSupportedError from .reflection import get_service_methods, get_module_name @@ -47,8 +48,14 @@ def __init__(self, service, thrift_module, def _make_request(self, method_name, args, kwargs): - endpoint = self._get_endpoint(method_name) result_type = self._get_result_type(method_name) + + if result_type is None: + raise OneWayNotSupportedError( + 'TChannel+Thrift does not currently support oneway procedues' + ) + + endpoint = self._get_endpoint(method_name) call_args = self._get_call_args(method_name, args, kwargs) request = ThriftRequest( @@ -75,7 +82,10 @@ def _get_args_type(self, method_name): def _get_result_type(self, method_name): - result_type = getattr(self.thrift_module, method_name + '_result') + # if None then result_type is oneway + result_type = getattr( + self.thrift_module, method_name + '_result', None + ) return result_type diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 179af75d..58e21daf 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -9,6 +9,7 @@ TChannel, from_thrift_module, schemes, response ) +from tchannel.errors import OneWayNotSupportedError from tchannel.tornado import TChannel as DeprecatedTChannel from tchannel.thrift import client_for from tests.data.generated.ThriftTest import ThriftTest, SecondService @@ -407,8 +408,31 @@ def testMultiException(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_oneway(): - # this is currently unsupported - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + # TODO - server should raise same exception as client + with pytest.raises(AssertionError): + @server.register(ThriftTest) + def testOneway(request, response, proxy): + pass + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + with pytest.raises(OneWayNotSupportedError): + yield tchannel.thrift(service.testOneway(1)) @pytest.mark.gen_test From 3d630282aa3dfeb993e0c42bb067f1428d7730ba Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 17:53:28 -0700 Subject: [PATCH 36/51] test basic thrift types --- tchannel/thrift/serializer.py | 1 - tests/schemes/test_thrift.py | 152 ++++++++++++++++++++++++++++++++-- 2 files changed, 147 insertions(+), 6 deletions(-) diff --git a/tchannel/thrift/serializer.py b/tchannel/thrift/serializer.py index 1bf4fdb2..83e85e93 100644 --- a/tchannel/thrift/serializer.py +++ b/tchannel/thrift/serializer.py @@ -42,7 +42,6 @@ def serialize_body(call_args): # result, TMemoryBuffer(body), (result_type, result_type.thrift_spec) # ) # - trans = TTransport.TMemoryBuffer() proto = TBinaryProtocol.TBinaryProtocolAccelerated(trans) call_args.write(proto) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 58e21daf..f0d0107b 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -113,31 +113,173 @@ def testString(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_byte(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testByte(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift( + service.testByte(63) + ) + + assert resp.headers == {} + assert resp.body == 63 @pytest.mark.gen_test @pytest.mark.call def test_i32(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testI32(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + # case #1 + resp = yield tchannel.thrift( + service.testI32(-1) + ) + assert resp.headers == {} + assert resp.body == -1 + + # case #2 + resp = yield tchannel.thrift( + service.testI32(1) + ) + assert resp.headers == {} + assert resp.body == 1 @pytest.mark.gen_test @pytest.mark.call def test_i64(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testI64(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift( + service.testI64(-34359738368) + ) + + assert resp.headers == {} + assert resp.body == -34359738368 @pytest.mark.gen_test @pytest.mark.call def test_double(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testDouble(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift( + service.testDouble(-5.235098235) + ) + + assert resp.headers == {} + assert resp.body == -5.235098235 @pytest.mark.gen_test @pytest.mark.call def test_binary(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testBinary(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift( + service.testBinary( + '\x0c\x00\x00\x0b\x00\x01\x00\x00\x00\x0bresp string\x00\x00' + ) + ) + + assert resp.headers == {} + assert ( + resp.body == + '\x0c\x00\x00\x0b\x00\x01\x00\x00\x00\x0bresp string\x00\x00' + ) @pytest.mark.gen_test From 679bded331d65a41f59e6a14a2efce39a96a3b38 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:03:39 -0700 Subject: [PATCH 37/51] thrift test nested structs --- tests/schemes/test_thrift.py | 50 +++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index f0d0107b..6cd77972 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -16,6 +16,11 @@ from tchannel.errors import ProtocolError +# TODO - where possible, in req/res style test, create parameterized tests, +# each test should test w headers and wout +# and potentially w retry and timeout as well. +# note this wont work with complex scenarios + @pytest.mark.gen_test @pytest.mark.call def test_void(): @@ -271,14 +276,15 @@ def testBinary(request, response, proxy): resp = yield tchannel.thrift( service.testBinary( - '\x0c\x00\x00\x0b\x00\x01\x00\x00\x00\x0bresp string\x00\x00' + # this is ThriftTest.Xtruct(string_thing='hi') + '\x0c\x00\x00\x0b\x00\x01\x00\x00\x00\x0bhi\x00\x00' ) ) assert resp.headers == {} assert ( resp.body == - '\x0c\x00\x00\x0b\x00\x01\x00\x00\x00\x0bresp string\x00\x00' + '\x0c\x00\x00\x0b\x00\x01\x00\x00\x00\x0bhi\x00\x00' ) @@ -375,7 +381,45 @@ def testStruct(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_nest(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testNest(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + xstruct = ThriftTest.Xtruct( + string_thing='hi', + byte_thing=1, + i32_thing=-1, + i64_thing=-34359738368, + ) + xstruct2 = ThriftTest.Xtruct2( + byte_thing=1, + struct_thing=xstruct, + i32_thing=1, + ) + + resp = yield tchannel.thrift( + service.testNest(thing=xstruct2) + ) + + assert resp.headers == {} + assert resp.body == xstruct2 @pytest.mark.gen_test From fc1514f8bc2d9eebd46b835a0889582dadd1f11a Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:08:19 -0700 Subject: [PATCH 38/51] thrift test map --- tests/schemes/test_thrift.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 6cd77972..90fd86d9 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -425,7 +425,40 @@ def testNest(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_map(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testMap(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + x = { + 0: 1, + 1: 2, + 2: 3, + 3: 4, + -1: -2, + } + + resp = yield tchannel.thrift( + service.testMap(thing=x) + ) + + assert resp.headers == {} + assert resp.body == x @pytest.mark.gen_test From 82c51a6ef279ef21258dc8499ef29cf7d91df2a5 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:10:13 -0700 Subject: [PATCH 39/51] thrift test string map --- tests/schemes/test_thrift.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 90fd86d9..cf77364a 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -464,7 +464,38 @@ def testMap(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_string_map(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testStringMap(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + x = { + 'hello': 'there', + 'my': 'name', + 'is': 'shirly', + } + + resp = yield tchannel.thrift( + service.testStringMap(thing=x) + ) + + assert resp.headers == {} + assert resp.body == x @pytest.mark.gen_test From 1685e08e12ef464d9dffc2468b82c92b7d335321 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:12:10 -0700 Subject: [PATCH 40/51] thrift test set --- tests/schemes/test_thrift.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index cf77364a..380b7c87 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -501,7 +501,34 @@ def testStringMap(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_set(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testSet(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + x = set([8, 1, 42]) + + resp = yield tchannel.thrift( + service.testSet(thing=x) + ) + + assert resp.headers == {} + assert resp.body == x @pytest.mark.gen_test From c4e47598302c551d67c3eee24c7f8ed055aaa00e Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:13:19 -0700 Subject: [PATCH 41/51] thrift test list --- tests/schemes/test_thrift.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 380b7c87..d56cffb0 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -534,7 +534,34 @@ def testSet(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_list(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testList(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + x = [1, 4, 9, -42] + + resp = yield tchannel.thrift( + service.testList(thing=x) + ) + + assert resp.headers == {} + assert resp.body == x @pytest.mark.gen_test From 79d883b8791428aea64a2f51e0062402f817b4a6 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:16:10 -0700 Subject: [PATCH 42/51] thrift test enum --- tests/schemes/test_thrift.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index d56cffb0..f1e38c84 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -567,7 +567,34 @@ def testList(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_enum(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testEnum(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + x = ThriftTest.Numberz.FIVE + + resp = yield tchannel.thrift( + service.testEnum(thing=x) + ) + + assert resp.headers == {} + assert resp.body == x @pytest.mark.gen_test From 3a7bf5510613f1747a264b3796f4f70f612d3586 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:20:06 -0700 Subject: [PATCH 43/51] thrift test type def --- tests/schemes/test_thrift.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index f1e38c84..20c08646 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -600,7 +600,34 @@ def testEnum(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_type_def(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testTypedef(request, response, proxy): + return request.args.thing + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + x = 0xffffffffffffff # 7 bytes of 0xff + + resp = yield tchannel.thrift( + service.testTypedef(thing=x) + ) + + assert resp.headers == {} + assert resp.body == x @pytest.mark.gen_test From 1bbb8bb9c57ddb85395195af4c7b6075037c5c72 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:29:33 -0700 Subject: [PATCH 44/51] thrift test map map --- tests/schemes/test_thrift.py | 43 +++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 20c08646..4119e9b5 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -633,7 +633,48 @@ def testTypedef(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_map_map(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + map_map = { + -4: { + -4: -4, + -3: -3, + -2: -2, + -1: -1, + }, + 4: { + 1: 1, + 2: 2, + 3: 3, + 4: 4, + }, + } + + @server.register(ThriftTest) + def testMapMap(request, response, proxy): + return map_map + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + resp = yield tchannel.thrift( + service.testMapMap(1) + ) + + assert resp.headers == {} + assert resp.body == map_map @pytest.mark.gen_test From 4966f2386d7f8d8c45bcf77c7e92c5e30de922cd Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:38:57 -0700 Subject: [PATCH 45/51] thrift test multi --- tests/schemes/test_thrift.py | 47 +++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 4119e9b5..63af66fb 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -686,7 +686,52 @@ def test_insanity(): @pytest.mark.gen_test @pytest.mark.call def test_multi(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testMulti(request, response, proxy): + return ThriftTest.Xtruct( + string_thing='Hello2', + byte_thing=request.args.arg0, + i32_thing=request.args.arg1, + i64_thing=request.args.arg2, + ) + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + x = ThriftTest.Xtruct( + string_thing='Hello2', + byte_thing=74, + i32_thing=0xff00ff, + i64_thing=0xffffffffd0d0 + ) + + resp = yield tchannel.thrift( + service.testMulti( + arg0=x.byte_thing, + arg1=x.i32_thing, + arg2=x.i64_thing, + arg3={0: 'abc'}, + arg4=ThriftTest.Numberz.FIVE, + arg5=0xf0f0f0, + ) + ) + + assert resp.headers == {} + assert resp.body == x @pytest.mark.gen_test From eab223f840595e0fab8118d6103990ab26c084e6 Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 19:57:03 -0700 Subject: [PATCH 46/51] thrift test insanity --- tests/schemes/test_thrift.py | 61 ++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 63af66fb..09ebbe30 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -680,7 +680,64 @@ def testMapMap(request, response, proxy): @pytest.mark.gen_test @pytest.mark.call def test_insanity(): - pass + + # Given this test server: + + server = DeprecatedTChannel(name='server') + + @server.register(ThriftTest) + def testInsanity(request, response, proxy): + result = { + 1: { + 2: request.args.argument, + 3: request.args.argument, + }, + 2: { + 6: ThriftTest.Insanity(), + }, + } + return result + + server.listen() + + # Make a call: + + tchannel = TChannel(name='client') + + service = from_thrift_module( + service='server', + thrift_module=ThriftTest, + hostport=server.hostport, + ) + + x = ThriftTest.Insanity( + userMap={ + ThriftTest.Numberz.EIGHT: 0xffffffffffffff, + }, + xtructs=[ + ThriftTest.Xtruct( + string_thing='Hello2', + byte_thing=74, + i32_thing=0xff00ff, + i64_thing=-34359738368, + ), + ], + ) + + resp = yield tchannel.thrift( + service.testInsanity(x) + ) + + assert resp.headers == {} + assert resp.body == { + 1: { + 2: x, + 3: x, + }, + 2: { + 6: ThriftTest.Insanity(), + }, + } @pytest.mark.gen_test @@ -716,7 +773,7 @@ def testMulti(request, response, proxy): string_thing='Hello2', byte_thing=74, i32_thing=0xff00ff, - i64_thing=0xffffffffd0d0 + i64_thing=0xffffffffd0d0, ) resp = yield tchannel.thrift( From 36dfc89ee7b35e8ef75ba73f9fd6bed41473665c Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 20:01:03 -0700 Subject: [PATCH 47/51] examples should be executable --- examples/simple/json/client.py | 0 examples/simple/json/server.py | 0 examples/simple/raw/client.py | 0 examples/simple/raw/server.py | 0 examples/simple/thrift/client.py | 0 examples/simple/thrift/server.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 examples/simple/json/client.py mode change 100644 => 100755 examples/simple/json/server.py mode change 100644 => 100755 examples/simple/raw/client.py mode change 100644 => 100755 examples/simple/raw/server.py mode change 100644 => 100755 examples/simple/thrift/client.py mode change 100644 => 100755 examples/simple/thrift/server.py diff --git a/examples/simple/json/client.py b/examples/simple/json/client.py old mode 100644 new mode 100755 diff --git a/examples/simple/json/server.py b/examples/simple/json/server.py old mode 100644 new mode 100755 diff --git a/examples/simple/raw/client.py b/examples/simple/raw/client.py old mode 100644 new mode 100755 diff --git a/examples/simple/raw/server.py b/examples/simple/raw/server.py old mode 100644 new mode 100755 diff --git a/examples/simple/thrift/client.py b/examples/simple/thrift/client.py old mode 100644 new mode 100755 diff --git a/examples/simple/thrift/server.py b/examples/simple/thrift/server.py old mode 100644 new mode 100755 From 8a6a4f383d3e7318dfa6d1dccd57657df496411a Mon Sep 17 00:00:00 2001 From: Grayson Koonce Date: Wed, 5 Aug 2015 20:08:55 -0700 Subject: [PATCH 48/51] travis you nasty --- examples/simple/thrift/server.py | 2 +- tests/test_examples.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple/thrift/server.py b/examples/simple/thrift/server.py index b1e330a7..94517fcc 100755 --- a/examples/simple/thrift/server.py +++ b/examples/simple/thrift/server.py @@ -5,7 +5,7 @@ from tests.data.generated.ThriftTest import ThriftTest -app = TChannel('thrift-server', hostport='localhost:54497') +app = TChannel('thrift-server', hostport='localhost:54444') @app.register(ThriftTest) diff --git a/tests/test_examples.py b/tests/test_examples.py index 5dbe38bc..9356c92d 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -67,7 +67,7 @@ def examples_dir(): ( ('raw', 'simple/raw/'), ('json', 'simple/json/'), - ('thrift', 'simple/thrift/'), + # ('thrift', 'simple/thrift/'), ('guide', 'guide/keyvalue/keyvalue/'), ) ) From 167dd5b106fcdd94820b80c8d61e0a316605869a Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Wed, 5 Aug 2015 22:37:41 -0700 Subject: [PATCH 49/51] Revert "examples should be executable" This reverts commit 36dfc89ee7b35e8ef75ba73f9fd6bed41473665c. --- examples/simple/json/client.py | 0 examples/simple/json/server.py | 0 examples/simple/raw/client.py | 0 examples/simple/raw/server.py | 0 examples/simple/thrift/client.py | 0 examples/simple/thrift/server.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 examples/simple/json/client.py mode change 100755 => 100644 examples/simple/json/server.py mode change 100755 => 100644 examples/simple/raw/client.py mode change 100755 => 100644 examples/simple/raw/server.py mode change 100755 => 100644 examples/simple/thrift/client.py mode change 100755 => 100644 examples/simple/thrift/server.py diff --git a/examples/simple/json/client.py b/examples/simple/json/client.py old mode 100755 new mode 100644 diff --git a/examples/simple/json/server.py b/examples/simple/json/server.py old mode 100755 new mode 100644 diff --git a/examples/simple/raw/client.py b/examples/simple/raw/client.py old mode 100755 new mode 100644 diff --git a/examples/simple/raw/server.py b/examples/simple/raw/server.py old mode 100755 new mode 100644 diff --git a/examples/simple/thrift/client.py b/examples/simple/thrift/client.py old mode 100755 new mode 100644 diff --git a/examples/simple/thrift/server.py b/examples/simple/thrift/server.py old mode 100755 new mode 100644 From b38e6edecf90244ea478428fab36e3e089f0a7bf Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Wed, 5 Aug 2015 22:37:44 -0700 Subject: [PATCH 50/51] Revert "travis you nasty" This reverts commit 8a6a4f383d3e7318dfa6d1dccd57657df496411a. --- examples/simple/thrift/server.py | 2 +- tests/test_examples.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple/thrift/server.py b/examples/simple/thrift/server.py index 94517fcc..b1e330a7 100644 --- a/examples/simple/thrift/server.py +++ b/examples/simple/thrift/server.py @@ -5,7 +5,7 @@ from tests.data.generated.ThriftTest import ThriftTest -app = TChannel('thrift-server', hostport='localhost:54444') +app = TChannel('thrift-server', hostport='localhost:54497') @app.register(ThriftTest) diff --git a/tests/test_examples.py b/tests/test_examples.py index 9356c92d..5dbe38bc 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -67,7 +67,7 @@ def examples_dir(): ( ('raw', 'simple/raw/'), ('json', 'simple/json/'), - # ('thrift', 'simple/thrift/'), + ('thrift', 'simple/thrift/'), ('guide', 'guide/keyvalue/keyvalue/'), ) ) From cc2e5f338ceebfa0532cc030f76e9e41f0119c65 Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Wed, 5 Aug 2015 20:22:33 -0700 Subject: [PATCH 51/51] fix thrift example test by packaging generated test data :/ --- CHANGES.rst | 11 +++++++++-- UPGRADE.rst | 7 +++++++ examples/simple/thrift/client.py | 4 +--- examples/simple/thrift/server.py | 6 +++--- setup.cfg | 6 +++++- .../generated => tchannel/testing/data}/__init__.py | 0 .../data/generated/ThriftTest/SecondService-remote | 0 .../data/generated/ThriftTest/SecondService.py | 0 .../data/generated/ThriftTest/ThriftTest-remote | 0 .../testing}/data/generated/ThriftTest/ThriftTest.py | 0 .../testing}/data/generated/ThriftTest/__init__.py | 0 .../testing}/data/generated/ThriftTest/constants.py | 0 .../testing}/data/generated/ThriftTest/ttypes.py | 0 tchannel/testing/data/generated/__init__.py | 0 tests/schemes/test_thrift.py | 3 ++- tests/test_examples.py | 12 +++++++++--- tests/thrift/test_module.py | 2 +- 17 files changed, 37 insertions(+), 14 deletions(-) rename {tests/data/generated => tchannel/testing/data}/__init__.py (100%) rename {tests => tchannel/testing}/data/generated/ThriftTest/SecondService-remote (100%) rename {tests => tchannel/testing}/data/generated/ThriftTest/SecondService.py (100%) rename {tests => tchannel/testing}/data/generated/ThriftTest/ThriftTest-remote (100%) rename {tests => tchannel/testing}/data/generated/ThriftTest/ThriftTest.py (100%) rename {tests => tchannel/testing}/data/generated/ThriftTest/__init__.py (100%) rename {tests => tchannel/testing}/data/generated/ThriftTest/constants.py (100%) rename {tests => tchannel/testing}/data/generated/ThriftTest/ttypes.py (100%) create mode 100644 tchannel/testing/data/generated/__init__.py diff --git a/CHANGES.rst b/CHANGES.rst index 7cbd2a10..7c0039fa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,10 +2,17 @@ Changelog ========= -0.14.1 (unreleased) +0.15.0 (unreleased) ------------------- -- Nothing changed yet. +- Introduced ``TChannel.call`` to replace the awkward ``request`` / ``send`` + calling pattern. +- Refactored arg schemes to no longer require an explicit use of + ``ArgSchemeBroker``. Use ``tchannel.json.call`` or ``tchanne.thrift.call`` to + send requests for a particular scheme. +- Fixed a bug where JSON arg2 (headers) was being returned a string instead of + a dictionary. +- Re-organized the examples directory. 0.14.0 (2015-08-03) diff --git a/UPGRADE.rst b/UPGRADE.rst index 8a2a53cd..6d5328bf 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -4,6 +4,13 @@ Version Upgrade Guide Migrating to a version of TChannel with breaking changes? This guide documents what broke and how to safely migrate to newer versions. +From 0.12 to 0.14 +----------------- + +- ``ArgSchemeBroker`` has been deprecated. +- ```` has been deprecated. + + From 0.11 to 0.12 ----------------- diff --git a/examples/simple/thrift/client.py b/examples/simple/thrift/client.py index 0e26ed16..ba22672c 100644 --- a/examples/simple/thrift/client.py +++ b/examples/simple/thrift/client.py @@ -1,10 +1,8 @@ import json from tornado import gen, ioloop - from tchannel import TChannel, from_thrift_module - -from tests.data.generated.ThriftTest import ThriftTest +from tchannel.testing.data.generated.ThriftTest import ThriftTest tchannel = TChannel('thrift-client') diff --git a/examples/simple/thrift/server.py b/examples/simple/thrift/server.py index b1e330a7..a93b9a2c 100644 --- a/examples/simple/thrift/server.py +++ b/examples/simple/thrift/server.py @@ -1,8 +1,8 @@ -from tornado import gen, ioloop +from __future__ import absolute_import +from tornado import gen, ioloop from tchannel.tornado import TChannel - -from tests.data.generated.ThriftTest import ThriftTest +from tchannel.testing.data.generated.ThriftTest import ThriftTest app = TChannel('thrift-server', hostport='localhost:54497') diff --git a/setup.cfg b/setup.cfg index 8f892cbc..9ea8197f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,4 +3,8 @@ create-wheel = yes [flake8] # Ignore files generated by Thrift -exclude = examples/guide/keyvalue/keyvalue/service/*,tchannel/zipkin/thrift/*,tchannel/testing/vcr/proxy/*,tests/data/generated/ThriftTest/* +exclude = + examples/guide/keyvalue/keyvalue/service/*, + tchannel/zipkin/thrift/*, + tchannel/testing/vcr/proxy/*, + tchannel/testing/data/generated/ThriftTest/* diff --git a/tests/data/generated/__init__.py b/tchannel/testing/data/__init__.py similarity index 100% rename from tests/data/generated/__init__.py rename to tchannel/testing/data/__init__.py diff --git a/tests/data/generated/ThriftTest/SecondService-remote b/tchannel/testing/data/generated/ThriftTest/SecondService-remote similarity index 100% rename from tests/data/generated/ThriftTest/SecondService-remote rename to tchannel/testing/data/generated/ThriftTest/SecondService-remote diff --git a/tests/data/generated/ThriftTest/SecondService.py b/tchannel/testing/data/generated/ThriftTest/SecondService.py similarity index 100% rename from tests/data/generated/ThriftTest/SecondService.py rename to tchannel/testing/data/generated/ThriftTest/SecondService.py diff --git a/tests/data/generated/ThriftTest/ThriftTest-remote b/tchannel/testing/data/generated/ThriftTest/ThriftTest-remote similarity index 100% rename from tests/data/generated/ThriftTest/ThriftTest-remote rename to tchannel/testing/data/generated/ThriftTest/ThriftTest-remote diff --git a/tests/data/generated/ThriftTest/ThriftTest.py b/tchannel/testing/data/generated/ThriftTest/ThriftTest.py similarity index 100% rename from tests/data/generated/ThriftTest/ThriftTest.py rename to tchannel/testing/data/generated/ThriftTest/ThriftTest.py diff --git a/tests/data/generated/ThriftTest/__init__.py b/tchannel/testing/data/generated/ThriftTest/__init__.py similarity index 100% rename from tests/data/generated/ThriftTest/__init__.py rename to tchannel/testing/data/generated/ThriftTest/__init__.py diff --git a/tests/data/generated/ThriftTest/constants.py b/tchannel/testing/data/generated/ThriftTest/constants.py similarity index 100% rename from tests/data/generated/ThriftTest/constants.py rename to tchannel/testing/data/generated/ThriftTest/constants.py diff --git a/tests/data/generated/ThriftTest/ttypes.py b/tchannel/testing/data/generated/ThriftTest/ttypes.py similarity index 100% rename from tests/data/generated/ThriftTest/ttypes.py rename to tchannel/testing/data/generated/ThriftTest/ttypes.py diff --git a/tchannel/testing/data/generated/__init__.py b/tchannel/testing/data/generated/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/schemes/test_thrift.py b/tests/schemes/test_thrift.py index 09ebbe30..5d87ff5c 100644 --- a/tests/schemes/test_thrift.py +++ b/tests/schemes/test_thrift.py @@ -12,7 +12,8 @@ from tchannel.errors import OneWayNotSupportedError from tchannel.tornado import TChannel as DeprecatedTChannel from tchannel.thrift import client_for -from tests.data.generated.ThriftTest import ThriftTest, SecondService +from tchannel.testing.data.generated.ThriftTest import SecondService +from tchannel.testing.data.generated.ThriftTest import ThriftTest from tchannel.errors import ProtocolError diff --git a/tests/test_examples.py b/tests/test_examples.py index 5dbe38bc..66313419 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -38,8 +38,11 @@ def popen(path, wait_for_listen=False): if wait_for_listen: # It would be more correct to check ``conn.status == # psutil.CONN_LISTEN`` but this works - while process.is_running() and not process.connections(): - pass + try: + while process.is_running() and not process.connections(): + pass + except psutil.Error: + raise AssertionError(process.stderr.read()) try: yield process @@ -94,7 +97,10 @@ def test_example(examples_dir, scheme, path): assert out == 'Hello, world!\n' return - body, headers = out.split(os.linesep)[:-1] + try: + body, headers = out.split(os.linesep)[:-1] + except ValueError: + raise AssertionError(out + client.stderr.read()) if scheme == 'raw': diff --git a/tests/thrift/test_module.py b/tests/thrift/test_module.py index 4e26b06b..49675e92 100644 --- a/tests/thrift/test_module.py +++ b/tests/thrift/test_module.py @@ -8,7 +8,7 @@ from tchannel import from_thrift_module from tchannel.thrift.module import ThriftRequestMaker, ThriftRequest -from tests.data.generated.ThriftTest import ThriftTest +from tchannel.testing.data.generated.ThriftTest import ThriftTest @pytest.mark.call