Skip to content
Browse files

modify parser to generate meta block in .json

  • Loading branch information...
1 parent 546678b commit bca7e6f86778cb4d4afb4e754876d43dd379a95b @coopernurse committed Apr 12, 2012
Showing with 138 additions and 31 deletions.
  1. +66 −1 barrister/parser.py
  2. +71 −29 barrister/test/parser_test.py
  3. +1 −1 conform/conform.json
View
67 barrister/parser.py
@@ -6,8 +6,24 @@
:copyright: 2012 by James Cooper.
:license: MIT, see LICENSE for more details.
"""
+
+import time
+import copy
+import operator
import cStringIO
from plex import Scanner, Lexicon, Str, State, IGNORE, Begin, Any, AnyBut, AnyChar, Range, Rep
+try:
+ import json
+except:
+ import simplejson as json
+
+def md5(s):
+ try:
+ import hashlib
+ return hashlib.md5(s).hexdigest()
+ except:
+ import md5
+ return md5.new(s).hexdigest()
native_types = [ "int", "float", "string", "bool" ]
letter = Range("AZaz")
@@ -19,7 +35,7 @@
comment = Str("// ") | Str("//")
type_opts = Str("[") + Rep(AnyBut("{}]\n")) + Str("]")
-def parse(idl_text, name=None, validate=True):
+def parse(idl_text, name=None, validate=True, add_meta=True):
if isinstance(idl_text, (str, unicode)):
idl_text = cStringIO.StringIO(idl_text)
@@ -28,6 +44,8 @@ def parse(idl_text, name=None, validate=True):
if validate:
scanner.validate()
if len(scanner.errors) == 0:
+ if add_meta:
+ scanner.add_meta()
return scanner.parsed
else:
raise IdlParseException(scanner.errors)
@@ -52,6 +70,53 @@ def eof(self):
if self.cur:
self.add_error("Unexpected end of file")
+ def add_meta(self):
+ import barrister
+ meta = {
+ "type" : "meta",
+ "barrister_version" : barrister.__version__,
+ "date_generated" : int(time.time() * 1000),
+ "checksum" : self.get_checksum()
+ }
+ self.parsed.append(meta)
+
+ def get_checksum(self):
+ """
+ Returns a checksum based on the IDL that ignores comments and ordering,
+ but detects changes to types, parameter order, and enum values.
+ """
+ arr = [ ]
+ for elem in self.parsed:
+ if elem["type"] == "struct":
+ s = ""
+ fields = copy.copy(elem["fields"])
+ fields.sort(key=operator.itemgetter("name"))
+ for f in fields:
+ s += "\t%s\t%s\t%s\t%s" % (f["name"], f["type"], f["is_array"], f["optional"])
+ arr.append("struct\t%s\t%s\t%s\n" % (elem["name"], elem["extends"], s))
+ elif elem["type"] == "enum":
+ s = "enum\t%s" % elem["name"]
+ vals = copy.copy(elem["values"])
+ vals.sort(key=operator.itemgetter("value"))
+ for v in vals: s += "\t%s" % v["value"]
+ s += "\n"
+ arr.append(s)
+ elif elem["type"] == "interface":
+ s = "interface\t%s" % elem["name"]
+ funcs = copy.copy(elem["functions"])
+ funcs.sort(key=operator.itemgetter("name"))
+ for f in funcs:
+ s += "[%s" % f["name"]
+ for p in f["params"]:
+ s += "\t%s\t%s" % (p["type"], p["is_array"])
+ ret = f["returns"]
+ s += "(%s\t%s\t%s)]" % (ret["type"], ret["is_array"], ret["optional"])
+ s += "\n"
+ arr.append(s)
+ arr.sort()
+ #print arr
+ return md5(json.dumps(arr))
+
def add_error(self, message, line=-1):
if line < 0:
(name, line, col) = self.position()
View
100 barrister/test/parser_test.py
@@ -11,6 +11,7 @@
:license: MIT, see LICENSE for more details.
"""
+import time
import unittest
from barrister.parser import parse, IdlParseException
@@ -35,7 +36,7 @@ def test_parse_comment(self):
{ "type" : "struct", "name" : "Person", "extends" : "",
"comment" : "this is a person", "fields" : [
field("age", "int") ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_parse_struct(self):
idl = """struct Person {
@@ -46,7 +47,7 @@ def test_parse_struct(self):
"type" : "struct",
"comment" : "", "extends" : "",
"fields" : [ field("email", "string"), field("age", "int") ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_parse_multiple(self):
idl = """struct Person { email string }
@@ -59,7 +60,7 @@ def test_parse_multiple(self):
"type" : "struct",
"comment" : "", "extends" : "",
"fields" : [ field("furry", "bool") ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_parse_enum(self):
idl = """enum Status { success fail
@@ -68,7 +69,7 @@ def test_parse_enum(self):
"values" : [ { "value" : "success", "comment" : "" },
{ "value" : "fail", "comment" : "" },
{ "value" : "invalid", "comment" : "" } ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_parse_interface(self):
idl = """interface MyService {
@@ -87,12 +88,12 @@ def test_parse_interface(self):
{ "name" : "login", "comment" : "",
"returns" : ret_field("LoginResponse"), "params" : [
{ "type" : "LoginRequest", "name" : "req", "is_array": False } ] } ] } ]
- self.assertEquals(expected, parse(idl, validate=False))
+ self.assertEquals(expected, parse(idl, add_meta=False, validate=False))
def test_invalid_struct(self):
idl = """struct foo { """
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 1, "message" : "Unexpected end of file" } ]
@@ -102,7 +103,7 @@ def test_missing_name(self):
idls = [ "struct {", "enum {", "interface { " ]
for idl in idls:
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 1, "message" : "Missing identifier" } ]
@@ -115,15 +116,15 @@ def test_enum_comments(self):
expected = [ { "name" : "Status", "type" : "enum", "comment" : "",
"values" : [ { "comment" : "Request successful",
"value": "success" } ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_array_type(self):
idl = """struct Animal {
friend_names []string }"""
expected = [ { "name" : "Animal", "type" : "struct", "comment" : "",
"extends" : "",
"fields" : [ field("friend_names", "string", "", True) ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_array_return_type(self):
idl = """interface FooService {
@@ -134,7 +135,7 @@ def test_array_return_type(self):
{ "name" : "repeat", "comment" : "",
"returns" : ret_field("string", True),
"params" : [ { "type" : "string", "name" : "s", "is_array": False } ] } ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_struct_comments(self):
idl = """struct Animal {
@@ -143,7 +144,7 @@ def test_struct_comments(self):
expected = [ { "name" : "Animal", "type" : "struct", "comment" : "",
"extends" : "",
"fields" : [ field("color", "string", "fur color") ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_function_comments(self):
idl = """interface FooService {
@@ -159,7 +160,7 @@ def test_function_comments(self):
"params" : [
{ "type" : "int", "name" : "a", "is_array": False },
{ "type" : "int", "name" : "b", "is_array": True } ] } ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_interface_comments(self):
idl = """// FooService is a..
@@ -173,7 +174,7 @@ def test_interface_comments(self):
{ "name" : "blah99", "returns" : ret_field("blah_Response"),
"comment" : "", "params" : [ ] }
] } ]
- self.assertEquals(expected, parse(idl, validate=False))
+ self.assertEquals(expected, parse(idl, add_meta=False, validate=False))
def test_extends_struct(self):
@@ -191,7 +192,7 @@ def test_extends_struct(self):
{ "name" : "Cat", "type" : "struct",
"extends" : "Animal", "comment" : "",
"fields" : [ field("purr_volume", "int") ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_no_dupe_types(self):
idl = """struct Animal {
@@ -211,7 +212,7 @@ def test_no_dupe_types(self):
do_other() bool
}"""
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 4, "message" : "type Animal already defined" },
@@ -227,7 +228,7 @@ def test_no_cycles(self):
residents []Animal
}"""
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 0, "message" : "cycle detected in: struct Animal" },
@@ -242,7 +243,7 @@ def test_cycle_detection(self):
toLoan []Book
toAck []Book
}"""
- parse(idl)
+ parse(idl, add_meta=False)
def test_interface_cant_be_field_type(self):
idl = """struct Animal {
@@ -252,7 +253,7 @@ def test_interface_cant_be_field_type(self):
do_something() bool
}"""
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 0, "message" : "interface FooService cannot be a field type" } ]
@@ -269,7 +270,7 @@ def test_types_exist(self):
a int
}"""
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 0, "message" : "undefined type: Color" },
@@ -290,7 +291,7 @@ def test_cant_override_parent_field(self):
gender bool
}"""
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 0, "message" : "Cat cannot redefine parent field color" },
@@ -303,7 +304,7 @@ def test_struct_cant_extend_enum(self):
color string
}"""
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 0, "message" : "Animal cannot extend enum Status" } ]
@@ -314,7 +315,7 @@ def test_struct_cant_extend_native_type(self):
color string
}"""
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 0, "message" : "Animal cannot extend float" } ]
@@ -323,7 +324,7 @@ def test_struct_cant_extend_native_type(self):
def test_struct_must_have_fields(self):
idl = "struct Animal { }"
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 1, "message" : "Animal must have at least one field" } ]
@@ -332,7 +333,7 @@ def test_struct_must_have_fields(self):
def test_interface_must_have_funcs(self):
idl = "interface FooService { }"
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 1, "message" : "FooService must have at least one function" } ]
@@ -341,7 +342,7 @@ def test_interface_must_have_funcs(self):
def test_enum_must_have_values(self):
idl = "enum Status { }"
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 1, "message" : "Status must have at least one value" } ]
@@ -356,7 +357,7 @@ def test_cycle_detection_for_interfaces(self):
subtract(a int, b int) BaseResponse
}"""
# should work - cycle detection should reset per function
- parse(idl)
+ parse(idl, add_meta=False)
def test_interface_cant_be_param(self):
idl = """interface BlargService {
@@ -367,7 +368,7 @@ def test_interface_cant_be_param(self):
subtract(a int, b int) BlargService
}"""
try:
- parse(idl)
+ parse(idl, add_meta=False)
self.fail("should have thrown exception")
except IdlParseException as e:
expected = [ { "line": 0, "message" : "interface BlargService cannot be a field type" },
@@ -383,7 +384,7 @@ def test_optional_struct_field(self):
"extends" : "", "comment" : "",
"fields" : [ field("firstName", "string"),
field("email", "string", optional=True) ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
def test_optional_return_type(self):
idl = """interface FooService {
@@ -393,7 +394,48 @@ def test_optional_return_type(self):
"functions" :
[ { "name" : "sayHi", "comment" : "", "params" : [ ],
"returns" : ret_field("string", optional=True) } ] } ]
- self.assertEquals(expected, parse(idl))
+ self.assertEquals(expected, parse(idl, add_meta=False))
+
+ def test_add_meta(self):
+ idl = """interface FooService {
+ do_foo() string
+}"""
+ start = int(time.time() * 1000)
+ parsed = parse(idl, add_meta=True)
+ meta = parsed[-1]
+ generated = meta["date_generated"]
+ checksum = meta["checksum"]
+ stop = int(time.time() * 1000)
+ self.assertTrue(generated >= start and generated <= stop)
+ self.assertTrue(checksum != None)
+
+ def test_meta_checksum(self):
+ base = "enum Y {\ndog\ncat\n}\nstruct Z {\n a int }\n"
+ base2 = "// foo\nstruct Z {\n //foo2\na int }\nenum Y {\ncat\ndog\n}\n"
+ equivalent = [ base+"interface FooService {\n do_foo() string\n}",
+ "interface FooService {\n do_foo() string\n}"+base2,
+ base+" interface FooService {\n // stuff\n do_foo() string\n\n}" ]
+ first_checksum = None
+ for idl in equivalent:
+ parsed = parse(idl, add_meta=True)
+ meta = parsed[-1]
+ checksum = meta["checksum"]
+ if first_checksum == None:
+ first_checksum = checksum
+ else:
+ self.assertEquals(first_checksum, checksum)
+
+ base3 = "enum Y {\ndog2\ncat\n}\nstruct Z {\n a int }\n"
+ base4 = "enum Y {\ndog\ncat\n}\nstruct Z {\n a float }\n"
+ different = [ base3+"interface FooService {\n do_foo() string\n}",
+ base4+"interface FooService {\n do_foo() string\n}",
+ base+"interface FooService {\n do_foo(a int) string\n}",
+ base+"interface FooService {\n do_foo(a int) string\n do_bar() int\n}",
+ base+"// foo interface\n interface FooService {\n // stuff\n do_foo() float\n\n}" ]
+ for idl in different:
+ parsed = parse(idl, add_meta=True)
+ meta = parsed[-1]
+ self.assertNotEquals(first_checksum, meta["checksum"])
if __name__ == "__main__":
unittest.main()
View
2 conform/conform.json
@@ -1 +1 @@
-[{"type": "comment", "value": "Barrister conformance IDL\n\nThe bits in here have silly names and the operations\nare not intended to be useful. The intent is to\nexercise as much of the IDL grammar as possible"}, {"comment": "", "values": [{"comment": "", "value": "ok"}, {"comment": "", "value": "err"}], "type": "enum", "name": "Status"}, {"comment": "", "values": [{"comment": "", "value": "add"}, {"comment": "", "value": "multiply"}], "type": "enum", "name": "MathOp"}, {"comment": "", "extends": "", "type": "struct", "name": "Response", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "Status", "name": "status"}]}, {"comment": "testing struct inheritance", "extends": "Response", "type": "struct", "name": "RepeatResponse", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "int", "name": "count"}, {"comment": "", "optional": false, "is_array": true, "type": "string", "name": "items"}]}, {"comment": "", "extends": "", "type": "struct", "name": "HiResponse", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "string", "name": "hi"}]}, {"comment": "", "extends": "", "type": "struct", "name": "RepeatRequest", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "string", "name": "to_repeat"}, {"comment": "", "optional": false, "is_array": false, "type": "int", "name": "count"}, {"comment": "", "optional": false, "is_array": false, "type": "bool", "name": "force_uppercase"}]}, {"comment": "", "extends": "", "type": "struct", "name": "Person", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "string", "name": "personId"}, {"comment": "", "optional": false, "is_array": false, "type": "string", "name": "firstName"}, {"comment": "", "optional": false, "is_array": false, "type": "string", "name": "lastName"}, {"comment": "", "optional": true, "is_array": false, "type": "string", "name": "email"}]}, {"comment": "", "functions": [{"comment": "returns a+b", "returns": {"optional": false, "is_array": false, "type": "int"}, "params": [{"is_array": false, "type": "int", "name": "a"}, {"is_array": false, "type": "int", "name": "b"}], "name": "add"}, {"comment": "performs the given operation against \nall the values in nums and returns the result", "returns": {"optional": false, "is_array": false, "type": "float"}, "params": [{"is_array": true, "type": "float", "name": "nums"}, {"is_array": false, "type": "MathOp", "name": "operation"}], "name": "calc"}, {"comment": "returns the square root of a", "returns": {"optional": false, "is_array": false, "type": "float"}, "params": [{"is_array": false, "type": "float", "name": "a"}], "name": "sqrt"}, {"comment": "Echos the req1.to_repeat string as a list,\noptionally forcing to_repeat to upper case\n\nRepeatResponse.items should be a list of strings\nwhose length is equal to req1.count", "returns": {"optional": false, "is_array": false, "type": "RepeatResponse"}, "params": [{"is_array": false, "type": "RepeatRequest", "name": "req1"}], "name": "repeat"}, {"comment": "returns a result with:\n hi=\"hi\" and status=\"ok\"", "returns": {"optional": false, "is_array": false, "type": "HiResponse"}, "params": [], "name": "say_hi"}, {"comment": "returns num as an array repeated 'count' number of times", "returns": {"optional": false, "is_array": true, "type": "int"}, "params": [{"is_array": false, "type": "int", "name": "num"}, {"is_array": false, "type": "int", "name": "count"}], "name": "repeat_num"}, {"comment": "simply returns p.personId\n\nwe use this to test the '[optional]' enforcement, \nas we invoke it with a null email", "returns": {"optional": false, "is_array": false, "type": "string"}, "params": [{"is_array": false, "type": "Person", "name": "p"}], "name": "putPerson"}], "type": "interface", "name": "A"}, {"comment": "a second interface to prove that the server dispatcher\nunderstands how to distinguish between interfaces in a contract", "functions": [{"comment": "simply returns s \nif s == \"return-null\" then you should return a null ", "returns": {"optional": true, "is_array": false, "type": "string"}, "params": [{"is_array": false, "type": "string", "name": "s"}], "name": "echo"}], "type": "interface", "name": "B"}]
+[{"type": "comment", "value": "Barrister conformance IDL\n\nThe bits in here have silly names and the operations\nare not intended to be useful. The intent is to\nexercise as much of the IDL grammar as possible"}, {"comment": "", "values": [{"comment": "", "value": "ok"}, {"comment": "", "value": "err"}], "type": "enum", "name": "Status"}, {"comment": "", "values": [{"comment": "", "value": "add"}, {"comment": "", "value": "multiply"}], "type": "enum", "name": "MathOp"}, {"comment": "", "extends": "", "type": "struct", "name": "Response", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "Status", "name": "status"}]}, {"comment": "testing struct inheritance", "extends": "Response", "type": "struct", "name": "RepeatResponse", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "int", "name": "count"}, {"comment": "", "optional": false, "is_array": true, "type": "string", "name": "items"}]}, {"comment": "", "extends": "", "type": "struct", "name": "HiResponse", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "string", "name": "hi"}]}, {"comment": "", "extends": "", "type": "struct", "name": "RepeatRequest", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "string", "name": "to_repeat"}, {"comment": "", "optional": false, "is_array": false, "type": "int", "name": "count"}, {"comment": "", "optional": false, "is_array": false, "type": "bool", "name": "force_uppercase"}]}, {"comment": "", "extends": "", "type": "struct", "name": "Person", "fields": [{"comment": "", "optional": false, "is_array": false, "type": "string", "name": "personId"}, {"comment": "", "optional": false, "is_array": false, "type": "string", "name": "firstName"}, {"comment": "", "optional": false, "is_array": false, "type": "string", "name": "lastName"}, {"comment": "", "optional": true, "is_array": false, "type": "string", "name": "email"}]}, {"comment": "", "functions": [{"comment": "returns a+b", "returns": {"optional": false, "is_array": false, "type": "int"}, "params": [{"is_array": false, "type": "int", "name": "a"}, {"is_array": false, "type": "int", "name": "b"}], "name": "add"}, {"comment": "performs the given operation against \nall the values in nums and returns the result", "returns": {"optional": false, "is_array": false, "type": "float"}, "params": [{"is_array": true, "type": "float", "name": "nums"}, {"is_array": false, "type": "MathOp", "name": "operation"}], "name": "calc"}, {"comment": "returns the square root of a", "returns": {"optional": false, "is_array": false, "type": "float"}, "params": [{"is_array": false, "type": "float", "name": "a"}], "name": "sqrt"}, {"comment": "Echos the req1.to_repeat string as a list,\noptionally forcing to_repeat to upper case\n\nRepeatResponse.items should be a list of strings\nwhose length is equal to req1.count", "returns": {"optional": false, "is_array": false, "type": "RepeatResponse"}, "params": [{"is_array": false, "type": "RepeatRequest", "name": "req1"}], "name": "repeat"}, {"comment": "returns a result with:\n hi=\"hi\" and status=\"ok\"", "returns": {"optional": false, "is_array": false, "type": "HiResponse"}, "params": [], "name": "say_hi"}, {"comment": "returns num as an array repeated 'count' number of times", "returns": {"optional": false, "is_array": true, "type": "int"}, "params": [{"is_array": false, "type": "int", "name": "num"}, {"is_array": false, "type": "int", "name": "count"}], "name": "repeat_num"}, {"comment": "simply returns p.personId\n\nwe use this to test the '[optional]' enforcement, \nas we invoke it with a null email", "returns": {"optional": false, "is_array": false, "type": "string"}, "params": [{"is_array": false, "type": "Person", "name": "p"}], "name": "putPerson"}], "type": "interface", "name": "A"}, {"comment": "a second interface to prove that the server dispatcher\nunderstands how to distinguish between interfaces in a contract", "functions": [{"comment": "simply returns s \nif s == \"return-null\" then you should return a null ", "returns": {"optional": true, "is_array": false, "type": "string"}, "params": [{"is_array": false, "type": "string", "name": "s"}], "name": "echo"}], "type": "interface", "name": "B"}, {"barrister_version": "0.1.0", "type": "meta", "date_generated": 1334261700516, "checksum": "34f6238ed03c6319017382e0fdc638a7"}]

0 comments on commit bca7e6f

Please sign in to comment.
Something went wrong with that request. Please try again.