From 18ffac79f67f8763c478b30adbd74eaa7988d323 Mon Sep 17 00:00:00 2001 From: James Cooper Date: Fri, 16 Mar 2012 16:02:39 -0700 Subject: [PATCH] Rearranged source tree to work as a package suitable to publishing to PyPI --- LICENSE | 22 ++ barrister/__init__.py | 19 + gen_docco.py => barrister/docco.py | 0 barrister/parser.py | 356 ++++++++++++++++++ runtime.py => barrister/runtime.py | 0 barrister/test/__init__.py | 10 + .../test/parser_test.py | 0 barrister/test/runtime_test.py | 210 +++++++++++ bin/barrister | 45 +++ run_tests.py | 8 + runtime_test.py | 2 +- 11 files changed, 671 insertions(+), 1 deletion(-) create mode 100644 LICENSE create mode 100644 barrister/__init__.py rename gen_docco.py => barrister/docco.py (100%) create mode 100644 barrister/parser.py rename runtime.py => barrister/runtime.py (100%) create mode 100644 barrister/test/__init__.py rename barrister_test.py => barrister/test/parser_test.py (100%) create mode 100644 barrister/test/runtime_test.py create mode 100755 bin/barrister create mode 100755 run_tests.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b661111 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +(The MIT License) + +Copyright (c) 2012 James Cooper + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/barrister/__init__.py b/barrister/__init__.py new file mode 100644 index 0000000..d074273 --- /dev/null +++ b/barrister/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" + barrister + ~~~~~~~~~ + + A RPC toolkit for building lightweight reliable services. Ideal for + both static and dynamic languages. + + :copyright: (c) 2012 by James Cooper. + :license: MIT, see LICENSE for more details. +""" +__version__ = '0.1.0' + +from .runtime import contract_from_file, idgen_uuid, idgen_seq +from .runtime import RpcException, Server, HttpTransport, InProcTransport +from .runtime import Client, Batch, BatchResult +from .runtime import Contract, Interface, Enum, Struct, Function +from .parser import parse, IdlParseException, IdlScanner +from .docco import docco_html diff --git a/gen_docco.py b/barrister/docco.py similarity index 100% rename from gen_docco.py rename to barrister/docco.py diff --git a/barrister/parser.py b/barrister/parser.py new file mode 100644 index 0000000..0b2749f --- /dev/null +++ b/barrister/parser.py @@ -0,0 +1,356 @@ +import cStringIO +from plex import Scanner, Lexicon, Str, State, IGNORE, Begin, Any, AnyChar, Range, Rep + +native_types = [ "int", "float", "string", "bool" ] +letter = Range("AZaz") +digit = Range("09") +under = Str("_") +ident = letter + Rep(letter | digit | under) +arr_ident = Str("[]") + ident +space = Any(" \t\n\r") +comment = Str("// ") | Str("//") + +def parse(idl_text, name=None, validate=True): + if isinstance(idl_text, (str, unicode)): + idl_text = cStringIO.StringIO(idl_text) + + scanner = IdlScanner(idl_text, name) + scanner.parse() + if validate: + scanner.validate() + if len(scanner.errors) == 0: + return scanner.parsed + else: + raise IdlParseException(scanner.errors) + +class IdlParseException(Exception): + + def __init__(self, errors): + Exception.__init__(self) + self.errors = errors + + def __str__(self): + s = "" + for e in self.errors: + if s != "": + s += ", " + s += "line: %d message: %s" % (e["line"], e["message"]) + return s + +class IdlScanner(Scanner): + + def eof(self): + if self.cur: + self.add_error("Unexpected end of file") + + def add_error(self, message, line=-1): + if line < 0: + (name, line, col) = self.position() + self.errors.append({"line": line, "message": message}) + + def begin_struct(self, text): + self.check_dupe_name(text) + self.cur = { "name" : text, "type" : "struct", "extends" : "", + "comment" : self.get_comment(), "fields" : [] } + self.begin('start-block') + + def begin_enum(self, text): + self.check_dupe_name(text) + self.cur = { "name" : text, "type" : "enum", + "comment" : self.get_comment(), "values" : [] } + self.begin('start-block') + + def begin_interface(self, text): + self.check_dupe_name(text) + self.cur = { "name" : text, "type" : "interface", + "comment" : self.get_comment(), "functions" : [] } + self.begin('start-block') + + def check_dupe_name(self, name): + if self.types.has_key(name): + self.add_error("type %s already defined" % name) + + def check_not_empty(self, cur, list_name, printable_name): + if len(cur[list_name]) == 0: + self.add_error("%s must have at least one %s" % (cur["name"], printable_name)) + return False + return True + + def start_block(self, text): + t = self.cur["type"] + if t == "struct": + self.begin("fields") + elif t == "enum": + self.begin("values") + elif t == "interface": + self.begin("functions") + else: + raise Exception("Invalid type: %s" % t) + + def end_block(self, text): + ok = False + t = self.cur["type"] + if t == "struct": + ok = self.check_not_empty(self.cur, "fields", "field") + elif t == "enum": + ok = self.check_not_empty(self.cur, "values", "value") + elif t == "interface": + ok = self.check_not_empty(self.cur, "functions", "function") + + if ok: + self.parsed.append(self.cur) + self.types[self.cur["name"]] = self.cur + + self.cur = None + self.begin('') + + def begin_field(self, text): + self.field = { "name" : text } + self.begin("field") + + def end_field(self, text): + self.field["type"] = text + self.field["comment"] = self.get_comment() + self.cur["fields"].append(self.field) + self.field = None + self.begin("fields") + + def begin_function(self, text): + self.function = { "name" : text, "comment" : self.get_comment(), "params" : [ ] } + self.begin("function-start") + + def begin_param(self, text): + self.param = { "name" : text } + self.begin("param") + + def end_param(self, text): + self.param["type"] = text + self.function["params"].append(self.param) + self.param = None + self.begin("end-param") + + def end_return(self, text): + self.function["returns"] = text + self.cur["functions"].append(self.function) + self.function = None + self.begin("end-function") + + def end_value(self, text): + if not text in self.cur["values"]: + val = { "value" : text, "comment" : self.get_comment() } + self.last_comment = "" + self.cur["values"].append(val) + + def get_comment(self): + comment = "" + if self.comment and len(self.comment) > 0: + comment = "".join(self.comment) + self.comment = None + return comment + + def start_comment(self, text): + if self.comment: + self.comment.append("\n") + else: + self.comment = [] + self.prev_state = self.state_name + self.begin("comment") + + def append_comment(self, text): + self.comment.append(text) + + def end_comment(self, text): + self.begin(self.prev_state) + self.prev_state = None + + def end_extends(self, text): + if self.cur and self.cur["type"] == "struct": + self.cur["extends"] = text + else: + self.add_error("extends is only supported for struct types") + + def add_comment_block(self, text): + comment = self.get_comment() + if comment: + self.parsed.append({"type" : "comment", "value" : comment}) + + lex = Lexicon([ + (Str("\n"), add_comment_block), + (space, IGNORE), + (Str('struct '), Begin('struct-start')), + (Str('enum '), Begin('enum-start')), + (Str('interface '), Begin('interface-start')), + (comment, start_comment), + State('struct-start', [ + (ident, begin_struct), + (space, IGNORE), + (AnyChar, "Missing identifier") ]), + State('enum-start', [ + (ident, begin_enum), + (space, IGNORE), + (AnyChar, "Missing identifier") ]), + State('interface-start', [ + (ident, begin_interface), + (space, IGNORE), + (AnyChar, "Missing identifier") ]), + State('start-block', [ + (space, IGNORE), + (Str("extends"), Begin('extends')), + (Str('{'), start_block) ]), + State('extends', [ + (space, IGNORE), + (ident, end_extends), + (Str('{'), start_block) ]), + State('fields', [ + (ident, begin_field), + (space, IGNORE), + (comment, start_comment), + (Str('{'), 'invalid'), + (Str('}'), end_block) ]), + State('field', [ + (ident, end_field), + (arr_ident, end_field), + (Str("\n"), 'invalid'), + (space, IGNORE), + (Str('{'), 'invalid'), + (Str('}'), 'invalid') ]), + State('functions', [ + (ident, begin_function), + (space, IGNORE), + (comment, start_comment), + (Str('{'), 'invalid'), + (Str('}'), end_block) ]), + State('function-start', [ + (Str("("), Begin('params')), + (Str("\n"), 'invalid'), + (space, IGNORE) ]), + State('params', [ + (ident, begin_param), + (space, IGNORE), + (Str(")"), Begin('function-return')) ]), + State('end-param', [ + (space, IGNORE), + (Str(","), Begin('params')), + (Str(")"), Begin('function-return')) ]), + State('param', [ + (ident, end_param), + (arr_ident, end_param), + (space, IGNORE) ]), + State('function-return', [ + (space, IGNORE), + (ident, end_return), + (arr_ident, end_return) ]), + State('end-function', [ + (Str("\n"), Begin('functions')), + (space, IGNORE) ]), + State('values', [ + (ident, end_value), + (space, IGNORE), + (comment, start_comment), + (Str('{'), 'invalid'), + (Str('}'), end_block) ]), + State('comment', [ + (Str("\n"), end_comment), + (AnyChar, append_comment) ]) + ]) + + def __init__(self, f, name): + Scanner.__init__(self, self.lex, f, name) + self.parsed = [ ] + self.errors = [ ] + self.types = { } + self.comment = None + self.cur = None + + def parse(self): + while True: + (t, name) = self.read() + if t is None: + break + else: + self.add_error(t) + break + + def validate_type(self, cur_type, types, level): + level += 1 + + cur_type = self.strip_array_chars(cur_type) + + if cur_type in native_types or cur_type in types: + pass + elif not self.types.has_key(cur_type): + self.add_error("undefined type: %s" % cur_type, line=0) + else: + cur = self.types[cur_type] + types.append(cur_type) + if cur["type"] == "struct": + if cur["extends"] != "": + self.validate_type(cur["extends"], types, level) + for f in cur["fields"]: + self.validate_type(f["type"], types, level) + elif cur["type"] == "interface": + # interface types must be top-level, so if len(types) > 1, we + # know this interface was used as a type in a function or struct + if level > 1: + msg = "interface %s cannot be a field type" % cur["name"] + self.add_error(msg, line=0) + else: + for f in cur["functions"]: + types = [ ] + for p in f["params"]: + self.validate_type(p["type"], types, 1) + self.validate_type(f["returns"], types, 1) + + def add_parent_fields(self, s, names, types): + if s["extends"] in native_types: + self.add_error("%s cannot extend %s" % (s["name"], s["extends"]), line=0) + elif self.types.has_key(s["extends"]): + if s["name"] not in types: + types.append(s["name"]) + parent = self.types[s["extends"]] + if parent["type"] == "struct": + for f in parent["fields"]: + if f["name"] not in names: + names.append(f["name"]) + self.add_parent_fields(parent, names, types) + else: + self.add_error("%s cannot extend %s %s" % (s["name"], parent["type"], parent["name"]), line=0) + + def validate_struct_extends(self, s): + names = [] + self.add_parent_fields(s, names, []) + for f in s["fields"]: + if f["name"] in names: + self.add_error("%s cannot redefine parent field %s" % (s["name"], f["name"]), line=0) + + def contains_cycle(self, name, types): + name = self.strip_array_chars(name) + if self.types.has_key(name): + t = self.types[name] + if t["type"] == "struct": + if name in types: + self.add_error("cycle detected in: %s %s" % (t["type"], name), line=0) + return True + else: + types.append(name) + if self.contains_cycle(t["extends"], types): + return True + for f in t["fields"]: + # use a copy of the type list to keep function checks separate + if self.contains_cycle(f["type"], types[:]): + return True + return False + + def strip_array_chars(self, name): + if name.find("[]") == 0: + return name[2:] + return name + + def validate(self): + for t in self.parsed: + if t["type"] == "comment": + pass + elif not self.contains_cycle(t["name"], []): + self.validate_type(t["name"], [], 0) + if t["type"] == "struct": + self.validate_struct_extends(t) diff --git a/runtime.py b/barrister/runtime.py similarity index 100% rename from runtime.py rename to barrister/runtime.py diff --git a/barrister/test/__init__.py b/barrister/test/__init__.py new file mode 100644 index 0000000..e1587f5 --- /dev/null +++ b/barrister/test/__init__.py @@ -0,0 +1,10 @@ +import unittest + +from .runtime_test import RuntimeTest +from .parser_test import ParserTest + +def all_tests(): + s = [ ] + s.append(unittest.TestLoader().loadTestsFromTestCase(ParserTest)) + s.append(unittest.TestLoader().loadTestsFromTestCase(RuntimeTest)) + return unittest.TestSuite(s) diff --git a/barrister_test.py b/barrister/test/parser_test.py similarity index 100% rename from barrister_test.py rename to barrister/test/parser_test.py diff --git a/barrister/test/runtime_test.py b/barrister/test/runtime_test.py new file mode 100644 index 0000000..7a6ed1b --- /dev/null +++ b/barrister/test/runtime_test.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python + +import uuid +import time +import unittest +import barrister +import runtime + +idl = """ +struct User { + userId string + password string + email string + emailVerified bool + dateCreated int + age float +} + +enum Status { + ok + invalid + error +} + +struct Response { + status Status + message string +} + +struct CountResponse extends Response { + count int +} + +struct CreateUserResponse extends Response { + userId string +} + +struct UserResponse extends Response { + user User +} + +struct UsersResponse extends Response { + users []User +} + +interface UserService { + get(userId string) UserResponse + create(user User) CreateUserResponse + update(user User) Response + validateEmail(userId string) Response + changePassword(userId string, oldPass string, newPass string) Response + countUsers() CountResponse + getAll(userIds []string) UsersResponse +} +""" + +def newUser(userId="abc123", email=None): + return { "userId" : userId, "password" : u"pw", "email" : email, + "emailVerified" : False, "dateCreated" : 1, "age" : 3.3 } + +def now_millis(): + return int(time.time() * 1000) + +class UserServiceImpl(object): + + def __init__(self): + self.users = { } + + def get(self, userId): + resp = self._resp("ok", "user created") + resp["user"] = self.users[userId] + return resp + + def create(self, user): + resp = self._resp("ok", "user created") + userId = uuid.uuid4().hex + user["dateCreated"] = now_millis() + resp["userId"] = userId + self.users[userId] = user + return resp + + def update(self, user): + userId = user["userId"] + self.users[userId] = user + return self._resp("ok", "user updated") + + def validateEmail(self, userId): + return self._resp("ok", "user updated") + + def changePassword(self, userId, oldPass, newPass): + return self._resp("ok", "password updated") + + def countUsers(self): + resp = self._resp("ok", "ok") + resp["count"] = len(self.users) + return resp + + def getAll(self, userIds): + return { "users": [] } + + def _resp(self, status, message): + return { "status" : status, "message" : message } + +class InProcTest(unittest.TestCase): + + def setUp(self): + contract = runtime.Contract(barrister.parse_str(idl)) + self.user_svc = UserServiceImpl() + self.server = runtime.Server(contract) + self.server.add_handler("UserService", self.user_svc) + + transport = runtime.InProcTransport(self.server) + self.client = runtime.Client(transport) + + def test_add_handler_invalid(self): + self.assertRaises(runtime.RpcException, self.server.add_handler, "foo", self.user_svc) + + def test_user_crud(self): + svc = self.client.UserService + user = newUser(email="foo@example.com") + resp = svc.create(user) + self.assertTrue(resp["userId"]) + user2 = svc.get(resp["userId"])["user"] + self.assertEquals(user["email"], user2["email"]) + self.assertTrue(user["dateCreated"] > 0) + self.assertEquals("ok", svc.changePassword("123", "oldpw", "newpw")["status"]) + self.assertEquals(1, svc.countUsers()["count"]) + svc.getAll([]) + + def test_invalid_req(self): + svc = self.client.UserService + cases = [ + [ svc.get ], # too few args + [ svc.get, 1, 2 ], # too many args + [ svc.get, 1 ], # wrong type + [ svc.create, None ], # wrong type + [ svc.create, 1 ], # wrong type + [ svc.create, { "UserId" : "1" } ], # unknown param + [ svc.create, { "userId" : 1 } ], # wrong type + [ svc.getAll, { } ], # wrong type + [ svc.getAll, [ 1 ] ] # wrong type + ] + for c in cases: + try: + if len(c) > 1: + c[0](c[1]) + else: + c[0]() + self.fail("Expected RpcException for: %s" % str(c)) + except runtime.RpcException: + pass + + def test_invalid_resp(self): + svc = self.client.UserService + responses = [ + { }, # missing fields + { "status" : "blah" }, # invalid enum + { "status" : "ok", "message" : 1 }, # invalid type + { "status" : "ok", "message" : "good", "blarg" : True }, # invalid field + { "status" : "ok", "message" : "good", "user" : { # missing password field + "userId" : "123", "email" : "foo@bar.com", + "emailVerified" : False, "dateCreated" : 1, "age" : 3.3 } }, + { "status" : "ok", "message" : "good", "user" : { # missing password field + "userId" : "123", "email" : "foo@bar.com", + "emailVerified" : False, "dateCreated" : 1, "age" : 3.3 } }, + { "status" : "ok", "user" : { # missing message field + "userId" : "123", "email" : "foo@bar.com", + "emailVerified" : False, "dateCreated" : 1, "age" : 3.3 } } + ] + for resp in responses: + self.user_svc.get = lambda id: resp + try: + svc.get("123") + self.fail("Expected RpcException for response: %s" % str(resp)) + except runtime.RpcException: + pass + + def test_batch(self): + batch = self.client.start_batch() + batch.UserService.create(newUser(userId="1")) + batch.UserService.create(newUser(userId="2")) + batch.UserService.countUsers() + result = batch.send() + self.assertEquals(3, result.count) + self.assertEquals(result.get(0)["message"], "user created") + self.assertEquals(result.get(1)["message"], "user created") + self.assertEquals(2, result.get(2)["count"]) + + def _test_bench(self): + start = time.time() + stop = start+1 + num = 0 + while time.time() < stop: + self.client.UserService.countUsers() + num += 1 + elapsed = time.time() - start + print "test_bench: num=%d microsec/op=%d" % (num, (elapsed*1000000)/num) + + start = time.time() + stop = start+1 + num = 0 + while time.time() < stop: + self.client.UserService.create(newUser()) + num += 1 + elapsed = time.time() - start + print "test_bench: num=%d microsec/op=%d" % (num, (elapsed*1000000)/num) + +if __name__ == "__main__": + unittest.main() + diff --git a/bin/barrister b/bin/barrister new file mode 100755 index 0000000..dc8e75f --- /dev/null +++ b/bin/barrister @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +from barrister import parse, docco_html +import optparse +import sys +try: + import json +except: + import simplejson as json + +if __name__ == "__main__": + parser = optparse.OptionParser("usage: %prog [options] [idl filename]") + parser.add_option("-i", "--stdin", dest="stdin", action="store_true", + default=False, help="Read IDL from STDIN") + parser.add_option("-d", "--docco", dest="docco", + default=None, type="string", + help="Generate docco HTML and save to this filename") + parser.add_option("-t", "--title", dest="title", + default=None, type="string", + help="title to use in generated HTML") + parser.add_option("-j", "--json", dest="json", + default=None, type="string", + help="File to write contract JSON to (defaults to STDOUT)") + (options, args) = parser.parse_args() + + if options.stdin: + parsed = parse(sys.stdin.read()) + elif len(args) < 1: + parser.error("Incorrect number of args") + else: + f = open(args[0]) + parsed = parse(f, args[0]) + f.close() + + if options.docco: + f = open(options.docco, "w") + f.write(docco_html(options.title, parsed)) + f.close() + + if options.json: + f = open(options.json, "w") + f.write(json.dumps(parsed)) + f.close() + else: + print json.dumps(parsed) diff --git a/run_tests.py b/run_tests.py new file mode 100755 index 0000000..791fc3c --- /dev/null +++ b/run_tests.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +import sys, os +import unittest + +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) +from barrister.test import all_tests + +unittest.TextTestRunner(verbosity=2).run(all_tests()) diff --git a/runtime_test.py b/runtime_test.py index 7a6ed1b..9e08e94 100644 --- a/runtime_test.py +++ b/runtime_test.py @@ -104,7 +104,7 @@ def _resp(self, status, message): class InProcTest(unittest.TestCase): def setUp(self): - contract = runtime.Contract(barrister.parse_str(idl)) + contract = runtime.Contract(barrister.parse(idl)) self.user_svc = UserServiceImpl() self.server = runtime.Server(contract) self.server.add_handler("UserService", self.user_svc)