Documentation of token_tree parse argument.

eerimoq · Jul 27, 2018 · 6ccaf53 · 6ccaf53
1 parent db9b627
commit 6ccaf53
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 24 deletions.
diff --git a/README.rst b/README.rst
@@ -36,6 +36,8 @@ The script:
 
 .. code-block:: python
 
+   from pprint import pprint
+
    import textparser
    from textparser import Sequence
 
@@ -56,16 +58,25 @@ The script:
 
 
    tree = Parser().parse('Hello, World!')
+   token_tree = Parser().parse('Hello, World!', token_tree=True)
 
    print('Tree:', tree)
+   print()
+   print('Token tree:')
+   pprint(token_tree)
 
 Script execution:
 
 .. code-block:: text
 
-   $ python3 examples/hello_world.py
+   $ env PYTHONPATH=. python3 examples/hello_world.py
    Tree: ['Hello', ',', 'World', '!']
-   $
+
+   Token tree:
+   [Token(kind='WORD', value='Hello', offset=0),
+    Token(kind=',', value=',', offset=5),
+    Token(kind='WORD', value='World', offset=7),
+    Token(kind='!', value='!', offset=12)]
 
 Contributing
 ============

diff --git a/docs/index.rst b/docs/index.rst
@@ -46,9 +46,6 @@ Functions and classes
 .. autoclass:: textparser.Optional
     :members:
 
-.. autoclass:: textparser.Inline
-    :members:
-
 .. autoclass:: textparser.Tag
     :members:
 

diff --git a/examples/hello_world.py b/examples/hello_world.py
@@ -1,8 +1,16 @@
 #!/usr/bin/env python3
 #
-# > env PYTHONPATH=.. python3 hello_world.py
+# $ env PYTHONPATH=.. python3 hello_world.py
 # Tree: ['Hello', ',', 'World', '!']
 #
+# Token tree:
+# [Token(kind='WORD', value='Hello', offset=0),
+#  Token(kind=',', value=',', offset=5),
+#  Token(kind='WORD', value='World', offset=7),
+#  Token(kind='!', value='!', offset=12)]
+#
+
+from pprint import pprint
 
 import textparser
 from textparser import Sequence
@@ -24,5 +32,9 @@ def grammar(self):
 
 
 tree = Parser().parse('Hello, World!')
+token_tree = Parser().parse('Hello, World!', token_tree=True)
 
 print('Tree:', tree)
+print()
+print('Token tree:')
+pprint(token_tree)
diff --git a/examples/proto3.py b/examples/proto3.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# > env PYTHONPATH=.. python3 proto3.py
+# $ env PYTHONPATH=.. python3 proto3.py
 # Tree: [['syntax', '=', '"proto3"', ';'],
 #  [['import', ['public'], '"foo.bar"', ';'],
 #   ['option', ['java_package', []], '=', '"com.example.foo"', ';'],

diff --git a/textparser.py b/textparser.py
@@ -512,6 +512,21 @@ def tokenize_init(spec):
 class Parser(object):
     """The abstract base class of all text parsers.
 
+    .. code-block:: python
+
+       >>> from textparser import Parser, Sequence
+       >>> class MyParser(Parser):
+       ...    def token_specs(self):
+       ...        return [
+       ...            ('SKIP',          r'[ \\r\\n\\t]+'),
+       ...            ('WORD',          r'\\w+'),
+       ...            ('EMARK',    '!', r'!'),
+       ...            ('COMMA',    ',', r','),
+       ...            ('MISMATCH',      r'.')
+       ...        ]
+       ...    def grammar(self):
+       ...        return Sequence('WORD', ',', 'WORD', '!')
+
     """
 
     def _unpack_token_specs(self):
@@ -542,17 +557,6 @@ def token_specs(self):
         ``(kind, name, re)``. If the second form is used, the grammar
         should use `name` instead of `kind`.
 
-        .. code-block:: python
-
-           def token_specs(self):
-               return [
-                   ('SKIP',          r'[ \\r\\n\\t]+'),
-                   ('WORD',          r'\\w+'),
-                   ('EMARK',    '!', r'!'),
-                   ('COMMA',    ',', r','),
-                   ('MISMATCH',      r'.')
-               ]
-
 
         """
 
@@ -596,11 +600,6 @@ def grammar(self):
         """The text grammar is used to create a parse tree out of a list of
         tokens.
 
-        .. code-block:: python
-
-           def grammar(self):
-               return Sequence('WORD', ',', 'WORD', '!')
-
 
         """
 
@@ -613,8 +612,15 @@ def parse(self, string, token_tree=False):
 
         .. code-block:: python
 
-           >>> Parser().parse('Hello, World!')
+           >>> MyParser().parse('Hello, World!')
            ['Hello', ',', 'World', '!']
+           >>> tree = MyParser().parse('Hello, World!', token_tree=True)
+           >>> from pprint import pprint
+           >>> pprint(tree)
+           [Token(kind='WORD', value='Hello', offset=0),
+            Token(kind=',', value=',', offset=5),
+            Token(kind='WORD', value='World', offset=7),
+            Token(kind='!', value='!', offset=12)]
 
         """