Parser started.

.travis.yml

@@ -5,6 +5,7 @@ python:
 
 install:
   - pip install coveralls
+  - pip install -r requirements.txt
 
 script:
   - coverage run --source=pbtools setup.py test

examples/address_book/generated/address_book.c

@@ -279,6 +279,11 @@ static bool decoder_peek_tag(struct decoder_t *self_p,
             && ((self_p->buf_p[self_p->pos] >> 3) == tag));
 }
 
+static int decoder_peek_tag_2(struct decoder_t *self_p)
+{
+    return (self_p->buf_p[self_p->pos] >> 3);
+}
+
 static uint64_t decoder_read_varint_value(struct decoder_t *self_p)
 {
     return (decoder_read_byte(self_p));
@@ -433,8 +438,23 @@ void address_book_person_phone_number_decode_inner(
     struct decoder_t *decoder_p,
     struct address_book_person_phone_number_t *phone_number_p)
 {
-    decoder_read_string(decoder_p, 1, &phone_number_p->number_p);
-    phone_number_p->type = decoder_read_varint(decoder_p, 2);
+    while (decoder_available(decoder_p)) {
+        switch (decoder_peek_tag_2(decoder_p)) {
+
+        case 1:
+            decoder_read_string(decoder_p, 1, &phone_number_p->number_p);
+            break;
+
+        case 2:
+            phone_number_p->type = decoder_read_varint(decoder_p, 2);
+            break;
+
+        default:
+            fprintf(stderr, "address_book_person_phone_number_decode_inner\n");
+            exit(1);
+            return;
+        }
+    }
 }
 
 void address_book_person_encode_inner(
@@ -468,21 +488,42 @@ void address_book_person_decode_inner(
     struct decoder_t decoder;
     int i;
 
-    decoder_read_string(decoder_p, 1, &person_p->name_p);
-    person_p->id = decoder_read_varint(decoder_p, 2);
-    decoder_read_string(decoder_p, 3, &person_p->email_p);
-    person_p->phones.length = decoder_peek_repeated_length(decoder_p, 4);
-    person_p->phones.items_p = heap_alloc(
-        decoder_p->heap_p,
-        sizeof(*phone_number_p) * person_p->phones.length);
-
-    for (i = 0; i < person_p->phones.length; i++) {
-        length = decoder_read_length_delimited(decoder_p, 4);
-        //fprintf(stderr, "repeated length: %llu\n", (unsigned long long)length);
-        decoder_init_slice(&decoder, decoder_p, length);
-        address_book_person_phone_number_decode_inner(&decoder,
-                                                      &person_p->phones.items_p[i]);
-        decoder_seek(decoder_p, decoder_get_result(&decoder));
+    while (decoder_available(decoder_p)) {
+        switch (decoder_peek_tag_2(decoder_p)) {
+
+        case 1:
+            decoder_read_string(decoder_p, 1, &person_p->name_p);
+            break;
+
+        case 2:
+            person_p->id = decoder_read_varint(decoder_p, 2);
+            break;
+
+        case 3:
+            decoder_read_string(decoder_p, 3, &person_p->email_p);
+            break;
+
+        case 4:
+            person_p->phones.length = decoder_peek_repeated_length(decoder_p, 4);
+            person_p->phones.items_p = heap_alloc(
+                decoder_p->heap_p,
+                sizeof(*phone_number_p) * person_p->phones.length);
+
+            for (i = 0; i < person_p->phones.length; i++) {
+                length = decoder_read_length_delimited(decoder_p, 4);
+                //fprintf(stderr, "repeated length: %llu\n", (unsigned long long)length);
+                decoder_init_slice(&decoder, decoder_p, length);
+                address_book_person_phone_number_decode_inner(&decoder,
+                                                              &person_p->phones.items_p[i]);
+                decoder_seek(decoder_p, decoder_get_result(&decoder));
+            }
+            break;
+
+        default:
+            fprintf(stderr, "address_book_person_decode_inner\n");
+            exit(1);
+            return;
+        }
     }
 }
 

pbtools/__init__.py

@@ -1 +1,2 @@
+from .parser import parse_file
 from .version import __version__

pbtools/parser.py

@@ -0,0 +1,159 @@
+import textparser
+from textparser import Sequence
+from textparser import ZeroOrMore
+from textparser import ZeroOrMoreDict
+from textparser import choice
+from textparser import Optional
+from textparser import NoMatch
+from textparser import Forward
+from textparser import Tag
+
+
+class Parser(textparser.Parser):
+
+    def token_specs(self):
+        return [
+            ('SKIP',          r'[ \r\n\t]+'),
+            ('SYNTAX',        r'syntax'),
+            ('PACKAGE',       r'package'),
+            ('MESSAGE',       r'message'),
+            ('REPEATED',      r'repeated'),
+            ('ENUM',          r'enum'),
+            ('LIDENT',        r'[a-zA-Z]\w*(\.[a-zA-Z]\w*)+'),
+            ('IDENT',         r'[a-zA-Z]\w*'),
+            ('INT',           r'0[xX][a-fA-F0-9]+|[0-9]+'),
+            ('PROTO3',        r'"proto3"'),
+            ('STRING',        r'"(\\"|[^"])*?"'),
+            ('DOT',      '.', r'\.'),
+            ('COMMA',    ',', r','),
+            ('EQ',       '=', r'='),
+            ('SCOLON',   ';', r';'),
+            ('LBRACE',   '{', r'{'),
+            ('RBRACE',   '}', r'}'),
+            ('MISMATCH',      r'.')
+        ]
+
+    def grammar(self):
+        ident = choice('IDENT',
+                       'SYNTAX',
+                       'PACKAGE',
+                       'MESSAGE',
+                       'REPEATED',
+                       'ENUM')
+        full_ident = choice(ident, 'LIDENT')
+        empty_statement = ';'
+
+        import_ = NoMatch()
+
+        package = Sequence('PACKAGE', full_ident, ';')
+
+        option = NoMatch()
+
+        enum_field = Sequence(ident, '=', 'INT', ';')
+        enum = Sequence('ENUM',
+                        ident,
+                        '{',
+                        ZeroOrMore(choice(enum_field, empty_statement)),
+                        '}')
+
+        field = Sequence(Optional('REPEATED'), ident, ident, '=', 'INT', ';')
+        message = Forward()
+        message <<= Sequence('MESSAGE',
+                             ident,
+                             '{',
+                             ZeroOrMore(choice(Tag('field', field),
+                                               enum,
+                                               message)),
+                             '}')
+
+        top_level_def = choice(message, enum)
+
+        syntax = Sequence('SYNTAX', '=', 'PROTO3', ';')
+
+        proto = Sequence(syntax,
+                         ZeroOrMoreDict(choice(import_,
+                                               package,
+                                               option,
+                                               top_level_def,
+                                               empty_statement)))
+
+        return proto
+
+
+class EnumField:
+
+    def __init__(self, tokens):
+        self.name = tokens[0]
+        self.tag = int(tokens[2])
+
+
+class Enum:
+
+    def __init__(self, tokens):
+        self.name = tokens[2]
+        self.fields = []
+
+        for item in tokens[3]:
+            self.fields.append(EnumField(item))
+
+
+class MessageField:
+
+    def __init__(self, tokens):
+        self.type = tokens[1]
+        self.name = tokens[2]
+        self.tag = int(tokens[4])
+        self.repeated = bool(tokens[0])
+
+
+class Message:
+
+    def __init__(self, tokens):
+        self.name = tokens[1]
+        self.fields = []
+        self.enums = []
+        self.messages = []
+
+        for item in tokens[3]:
+            kind = item[0]
+
+            if kind == 'field':
+                self.fields.append(MessageField(item[1]))
+            elif kind == 'enum':
+                self.enums.append(Enum(item))
+            elif kind == 'message':
+                self.messages.append(Message(item))
+            else:
+                raise RuntimeError()
+
+
+def load_package(tokens):
+    try:
+        return tokens[1]['package'][0][1]
+    except KeyError:
+        raise RuntimeError()
+
+
+def load_messages(tokens):
+    messages = []
+
+    for message in tokens[1].get('message', []):
+        messages.append(Message(message))
+
+    return messages
+
+
+class Proto:
+
+    def __init__(self, tree):
+        self.package = load_package(tree)
+        self.messages = load_messages(tree)
+
+
+def parse_string(text):
+    return Proto(Parser().parse(text))
+
+
+def parse_file(filename):
+    with open(filename, 'r') as fin:
+        return parse_string(fin.read())

requirements.txt

@@ -0,0 +1 @@
+textparser

setup.py

@@ -25,6 +25,9 @@ def find_version():
       keywords=['protobuf', 'proto', 'protocol buffers'],
       url='https://github.com/eerimoq/pbtools',
       packages=find_packages(exclude=['tests']),
+      install_requires=[
+          'textparser>=0.21.1'
+      ],
       test_suite="tests",
       entry_points = {
           'console_scripts': ['pbtools=pbtools.__init__:_main']

tests/files/address_book.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package address_book;
+
+message Person {
+    string name = 1;
+    int32 id = 2;
+    string email = 3;
+
+    enum PhoneType {
+        MOBILE = 0;
+        HOME = 1;
+        WORK = 2;
+    }
+
+    message PhoneNumber {
+        string number = 1;
+        PhoneType type = 2;
+    }
+
+    repeated PhoneNumber phones = 4;
+}
+
+message AddressBook {
+    repeated Person people = 1;
+}