Skip to content

Commit

Permalink
Parser continued.
Browse files Browse the repository at this point in the history
  • Loading branch information
@eerimoq
eerimoq committed Aug 22, 2019
1 parent fdca9c6 commit 9bcef29
Show file tree
Hide file tree
Showing 3 changed files with 133 additions and 22 deletions.
115 changes: 93 additions & 22 deletions pbtools/parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,7 @@ class Parser(textparser.Parser):

def token_specs(self):
return [
('SKIP', r'[ \r\n\t]+'),
('SYNTAX', r'syntax'),
('PACKAGE', r'package'),
('MESSAGE', r'message'),
('REPEATED', r'repeated'),
('ENUM', r'enum'),
('SKIP', r'[ \r\n\t]+|//.*?\n'),
('LIDENT', r'[a-zA-Z]\w*(\.[a-zA-Z]\w*)+'),
('IDENT', r'[a-zA-Z]\w*'),
('INT', r'0[xX][a-fA-F0-9]+|[0-9]+'),
Expand All @@ -30,45 +25,82 @@ def token_specs(self):
('SCOLON', ';', r';'),
('LBRACE', '{', r'{'),
('RBRACE', '}', r'}'),
('LPAREN', '(', r'\('),
('RPAREN', ')', r'\)'),
('MISMATCH', r'.')
]

def keywords(self):
return set([
'syntax',
'package',
'message',
'repeated',
'enum',
'service',
'rpc',
'returns',
'stream',
'import'
])

def grammar(self):
ident = choice('IDENT',
'SYNTAX',
'PACKAGE',
'MESSAGE',
'REPEATED',
'ENUM')
'syntax',
'package',
'message',
'repeated',
'enum',
'service',
'rpc',
'returns',
'stream',
'import')
full_ident = choice(ident, 'LIDENT')
empty_statement = ';'
message_type = Sequence(Optional('.'), full_ident)

import_ = NoMatch()
import_ = Sequence('import',
Optional(choice('weak', 'public')),
'STRING')

package = Sequence('PACKAGE', full_ident, ';')
package = Sequence('package', full_ident, ';')

option = NoMatch()

enum_field = Sequence(ident, '=', 'INT', ';')
enum = Sequence('ENUM',
enum = Sequence('enum',
ident,
'{',
ZeroOrMore(choice(enum_field, empty_statement)),
'}')

field = Sequence(Optional('REPEATED'), ident, ident, '=', 'INT', ';')
field = Sequence(Optional('repeated'), message_type, ident, '=', 'INT', ';')
message = Forward()
message <<= Sequence('MESSAGE',
message <<= Sequence('message',
ident,
'{',
ZeroOrMore(choice(Tag('field', field),
enum,
message)),
'}')

top_level_def = choice(message, enum)
rpc = Sequence('rpc',
ident,
'(', Optional('stream'), message_type, ')',
'returns',
'(', Optional('stream'), message_type, ')',
';')

service = Sequence('service',
ident,
'{',
ZeroOrMore(choice(option, rpc, empty_statement)),
'}')

syntax = Sequence('SYNTAX', '=', 'PROTO3', ';')
top_level_def = choice(message, enum, service)

syntax = Sequence('syntax', '=', 'PROTO3', ';')

proto = Sequence(syntax,
ZeroOrMoreDict(choice(import_,
Expand All @@ -80,6 +112,10 @@ def grammar(self):
return proto


def load_message_type(tokens):
return tokens[1]


class EnumField:

def __init__(self, tokens):
Expand All @@ -95,16 +131,16 @@ def __init__(self, tokens):

for item in tokens[3]:
self.fields.append(EnumField(item))


class MessageField:

def __init__(self, tokens):
self.type = tokens[1]
self.type = load_message_type(tokens[1])
self.name = tokens[2]
self.tag = int(tokens[4])
self.repeated = bool(tokens[0])


class Message:

Expand All @@ -124,7 +160,32 @@ def __init__(self, tokens):
elif kind == 'message':
self.messages.append(Message(item))
else:
raise RuntimeError()
raise RuntimeError(kind)


class Rpc:

def __init__(self, tokens):
self.name = tokens[1]
self.request_type = tokens[4][1]
self.request_stream = False
self.response_type = tokens[9][1]
self.response_stream = False


class Service:

def __init__(self, tokens):
self.name = tokens[1]
self.rpcs = []

for item in tokens[3]:
kind = item[0]

if kind == 'rpc':
self.rpcs.append(Rpc(item))
else:
raise RuntimeError(kind)


def load_package(tokens):
Expand All @@ -143,11 +204,21 @@ def load_messages(tokens):
return messages


def load_services(tokens):
services = []

for service in tokens[1].get('service', []):
services.append(Service(service))

return services


class Proto:

def __init__(self, tree):
self.package = load_package(tree)
self.messages = load_messages(tree)
self.services = load_services(tree)


def parse_string(text):
Expand Down
16 changes: 16 additions & 0 deletions tests/files/service.proto
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
syntax = "proto3";

package service;

message Request {
string value = 1;
}

message Response {
string value = 1;
}

service MyService {
rpc Foo (Request) returns (Response);
rpc Bar (Request) returns (Response);
}
24 changes: 24 additions & 0 deletions tests/test_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,30 @@ def test_address_book(self):
self.assertEqual(field.tag, 1)
self.assertTrue(field.repeated)

def test_service(self):
parsed = pbtools.parse_file('tests/files/service.proto')

self.assertEqual(parsed.package, 'service')
self.assertEqual(len(parsed.messages), 2)
self.assertEqual(len(parsed.services), 1)

service = parsed.services[0]
self.assertEqual(len(service.rpcs), 2)

rpc = service.rpcs[0]
self.assertEqual(rpc.name, 'Foo')
self.assertEqual(rpc.request_type, 'Request')
self.assertFalse(rpc.request_stream)
self.assertEqual(rpc.response_type, 'Response')
self.assertFalse(rpc.response_stream)

rpc = service.rpcs[1]
self.assertEqual(rpc.name, 'Bar')
self.assertEqual(rpc.request_type, 'Request')
self.assertFalse(rpc.request_stream)
self.assertEqual(rpc.response_type, 'Response')
self.assertFalse(rpc.response_stream)


if __name__ == '__main__':
unittest.main()

0 comments on commit 9bcef29

Please sign in to comment.