Another attempt at fixing tests

hipster-philology · Feb 25, 2020 · 680d205 · 680d205
1 parent adff872
commit 680d205
Show file tree

Hide file tree

Showing 3 changed files with 151 additions and 119 deletions.
diff --git a/tests/test_models/crazy_text_file-pie.txt b/tests/test_models/crazy_text_file-pie.txt
@@ -1,55 +1,59 @@
 token	lemma	pos	morph
-\	\	PUNC	MORPH=empty
-\	\	PUNC	MORPH=empty
-<	<	PUNC	MORPH=empty
-1	lemma0	pos0	Case=Case0|Numb=Numb0|Deg=Deg0|Mood=Mood0|Tense=Tense0|Voice=Voice0|Person=Person0
->	>	PUNC	MORPH=empty
-[	[	PUNC	MORPH=empty
-$	$	PUNC	MORPH=empty
-@	@	PUNC	MORPH=empty
-$	$	PUNC	MORPH=empty
-]	]	PUNC	MORPH=empty
-(	(	PUNC	MORPH=empty
-v	lemma1	pos1	Case=Case1|Numb=Numb1|Deg=Deg1|Mood=Mood1|Tense=Tense1|Voice=Voice1|Person=Person1
-)	)	PUNC	MORPH=empty
-\	\	PUNC	MORPH=empty
-\	\	PUNC	MORPH=empty
-§	§	PUNC	MORPH=empty
-\	\	PUNC	MORPH=empty
-<	<	PUNC	MORPH=empty
-1	lemma2	pos2	Case=Case2|Numb=Numb2|Deg=Deg2|Mood=Mood2|Tense=Tense2|Voice=Voice2|Person=Person2
->	>	PUNC	MORPH=empty
-[	[	PUNC	MORPH=empty
-$	$	PUNC	MORPH=empty
-@	@	PUNC	MORPH=empty
-$	$	PUNC	MORPH=empty
-]	]	PUNC	MORPH=empty
-\	\	PUNC	MORPH=empty
-§	§	PUNC	MORPH=empty
-§	§	PUNC	MORPH=empty
-\	\	PUNC	MORPH=empty
-[	[	PUNC	MORPH=empty
-i	lemma3	pos3	Case=Case3|Numb=Numb3|Deg=Deg3|Mood=Mood3|Tense=Tense3|Voice=Voice3|Person=Person3
-]	]	PUNC	MORPH=empty
-\	\	PUNC	MORPH=empty
-§	§	PUNC	MORPH=empty
-en	lemma4	pos4	Case=Case4|Numb=Numb4|Deg=Deg4|Mood=Mood4|Tense=Tense4|Voice=Voice4|Person=Person4
-honor	lemma5	pos5	Case=Case5|Numb=Numb5|Deg=Deg5|Mood=Mood5|Tense=Tense5|Voice=Voice5|Person=Person5
-et	lemma6	pos6	Case=Case6|Numb=Numb6|Deg=Deg6|Mood=Mood6|Tense=Tense6|Voice=Voice6|Person=Person6
-en	lemma7	pos7	Case=Case7|Numb=Numb7|Deg=Deg7|Mood=Mood7|Tense=Tense7|Voice=Voice7|Person=Person7
-bie	lemma8	pos8	Case=Case8|Numb=Numb8|Deg=Deg8|Mood=Mood8|Tense=Tense8|Voice=Voice8|Person=Person8
--ne	ne2	pos9	Case=Case9|Numb=Numb9|Deg=Deg9|Mood=Mood9|Tense=Tense9|Voice=Voice9|Person=Person9
-et	lemma10	pos10	Case=Case10|Numb=Numb10|Deg=Deg10|Mood=Mood10|Tense=Tense10|Voice=Voice10|Person=Person10
-en	lemma11	pos11	Case=Case11|Numb=Numb11|Deg=Deg11|Mood=Mood11|Tense=Tense11|Voice=Voice11|Person=Person11
-gra	lemma12	pos12	Case=Case12|Numb=Numb12|Deg=Deg12|Mood=Mood12|Tense=Tense12|Voice=Voice12|Person=Person12
--ne	ne2	pos13	Case=Case13|Numb=Numb13|Deg=Deg13|Mood=Mood13|Tense=Tense13|Voice=Voice13|Person=Person13
-remembrançe	lemma14	pos14	Case=Case14|Numb=Numb14|Deg=Deg14|Mood=Mood14|Tense=Tense14|Voice=Voice14|Person=Person14
-§	§	PUNC	MORPH=empty
-et	lemma15	pos15	Case=Case15|Numb=Numb15|Deg=Deg15|Mood=Mood15|Tense=Tense15|Voice=Voice15|Person=Person15
-offerant	lemma16	pos16	Case=Case16|Numb=Numb16|Deg=Deg16|Mood=Mood16|Tense=Tense16|Voice=Voice16|Person=Person16
-mercé	lemma17	pos17	Case=Case17|Numb=Numb17|Deg=Deg17|Mood=Mood17|Tense=Tense17|Voice=Voice17|Person=Person17
-,	,	PUNC	MORPH=empty
-honor	lemma18	pos18	Case=Case18|Numb=Numb18|Deg=Deg18|Mood=Mood18|Tense=Tense18|Voice=Voice18|Person=Person18
-et	lemma19	pos19	Case=Case19|Numb=Numb19|Deg=Deg19|Mood=Mood19|Tense=Tense19|Voice=Voice19|Person=Person19
-celebrançe	lemma20	pos20	Case=Case20|Numb=Numb20|Deg=Deg20|Mood=Mood20|Tense=Tense20|Voice=Voice20|Person=Person20
-§	§	PUNC	MORPH=empty
+\	\	PUNC	MORPH=empty	\
+\	\	PUNC	MORPH=empty	\
+\	\	PUNC	MORPH=empty	\
+<	<	PUNC	MORPH=empty	<
+1	lemma0	pos0	Case=Case0|Numb=Numb0|Deg=Deg0|Mood=Mood0|Tense=Tense0|Voice=Voice0|Person=Person0	1
+>	>	PUNC	MORPH=empty	>
+[	[	PUNC	MORPH=empty	[
+$	$	PUNC	MORPH=empty	$
+@	@	PUNC	MORPH=empty	@
+$	$	PUNC	MORPH=empty	$
+]	]	PUNC	MORPH=empty	]
+(	(	PUNC	MORPH=empty	(
+v	lemma1	pos1	Case=Case1|Numb=Numb1|Deg=Deg1|Mood=Mood1|Tense=Tense1|Voice=Voice1|Person=Person1	u
+)	)	PUNC	MORPH=empty	)
+\	\	PUNC	MORPH=empty	\
+\	\	PUNC	MORPH=empty	\
+\	\	PUNC	MORPH=empty	\
+§	§	PUNC	MORPH=empty	§
+\	\	PUNC	MORPH=empty	\
+\	\	PUNC	MORPH=empty	\
+<	<	PUNC	MORPH=empty	<
+1	lemma2	pos2	Case=Case2|Numb=Numb2|Deg=Deg2|Mood=Mood2|Tense=Tense2|Voice=Voice2|Person=Person2	1
+>	>	PUNC	MORPH=empty	>
+[	[	PUNC	MORPH=empty	[
+$	$	PUNC	MORPH=empty	$
+@	@	PUNC	MORPH=empty	@
+$	$	PUNC	MORPH=empty	$
+]	]	PUNC	MORPH=empty	]
+\	\	PUNC	MORPH=empty	\
+\	\	PUNC	MORPH=empty	\
+§	§	PUNC	MORPH=empty	§
+§	§	PUNC	MORPH=empty	§
+\	\	PUNC	MORPH=empty	\
+[	[	PUNC	MORPH=empty	[
+i	lemma3	pos3	Case=Case3|Numb=Numb3|Deg=Deg3|Mood=Mood3|Tense=Tense3|Voice=Voice3|Person=Person3	i
+]	]	PUNC	MORPH=empty	]
+\	\	PUNC	MORPH=empty	\
+§	§	PUNC	MORPH=empty	§
+en	lemma4	pos4	Case=Case4|Numb=Numb4|Deg=Deg4|Mood=Mood4|Tense=Tense4|Voice=Voice4|Person=Person4	en
+honor	lemma5	pos5	Case=Case5|Numb=Numb5|Deg=Deg5|Mood=Mood5|Tense=Tense5|Voice=Voice5|Person=Person5	honor
+et	lemma6	pos6	Case=Case6|Numb=Numb6|Deg=Deg6|Mood=Mood6|Tense=Tense6|Voice=Voice6|Person=Person6	et
+en	lemma7	pos7	Case=Case7|Numb=Numb7|Deg=Deg7|Mood=Mood7|Tense=Tense7|Voice=Voice7|Person=Person7	en
+bie	lemma8	pos8	Case=Case8|Numb=Numb8|Deg=Deg8|Mood=Mood8|Tense=Tense8|Voice=Voice8|Person=Person8	bie
+-ne	ne2	pos9	Case=Case9|Numb=Numb9|Deg=Deg9|Mood=Mood9|Tense=Tense9|Voice=Voice9|Person=Person9	-ne
+et	lemma10	pos10	Case=Case10|Numb=Numb10|Deg=Deg10|Mood=Mood10|Tense=Tense10|Voice=Voice10|Person=Person10	et
+en	lemma11	pos11	Case=Case11|Numb=Numb11|Deg=Deg11|Mood=Mood11|Tense=Tense11|Voice=Voice11|Person=Person11	en
+gra	lemma12	pos12	Case=Case12|Numb=Numb12|Deg=Deg12|Mood=Mood12|Tense=Tense12|Voice=Voice12|Person=Person12	gra
+-ne	ne2	pos13	Case=Case13|Numb=Numb13|Deg=Deg13|Mood=Mood13|Tense=Tense13|Voice=Voice13|Person=Person13	-ne
+remembrançe	lemma14	pos14	Case=Case14|Numb=Numb14|Deg=Deg14|Mood=Mood14|Tense=Tense14|Voice=Voice14|Person=Person14	remembrançe
+§	§	PUNC	MORPH=empty	§
+et	lemma15	pos15	Case=Case15|Numb=Numb15|Deg=Deg15|Mood=Mood15|Tense=Tense15|Voice=Voice15|Person=Person15	et
+offerant	lemma16	pos16	Case=Case16|Numb=Numb16|Deg=Deg16|Mood=Mood16|Tense=Tense16|Voice=Voice16|Person=Person16	offerant
+mercé	lemma17	pos17	Case=Case17|Numb=Numb17|Deg=Deg17|Mood=Mood17|Tense=Tense17|Voice=Voice17|Person=Person17	mercé
+,	,	PUNC	MORPH=empty	,
+honor	lemma18	pos18	Case=Case18|Numb=Numb18|Deg=Deg18|Mood=Mood18|Tense=Tense18|Voice=Voice18|Person=Person18	honor
+et	lemma19	pos19	Case=Case19|Numb=Numb19|Deg=Deg19|Mood=Mood19|Tense=Tense19|Voice=Voice19|Person=Person19	et
+celebrançe	lemma20	pos20	Case=Case20|Numb=Numb20|Deg=Deg20|Mood=Mood20|Tense=Tense20|Voice=Voice20|Person=Person20	celebrançe
+§	§	PUNC	MORPH=empty	§
diff --git a/tests/test_models/crazy_text_file.txt b/tests/test_models/crazy_text_file.txt
@@ -1,5 +1,5 @@
-\\<1>[$@$](V)\\§
-\<1>[$@$]\§
+\\\<1>[$@$](V)\\\§
+\\<1>[$@$]\\§
 §
 \[I]\§
 En honor et en bien et en gran remembrançe §

diff --git a/tests/test_models/test_lasla.py b/tests/test_models/test_lasla.py
@@ -2,6 +2,7 @@
 from pie_extended.models import lasla
 from pie_extended.testing_utils import FakeTagger, create_auto_tagger
 from typing import List, Tuple
+import csv
 
 from unittest import TestCase
 
@@ -143,67 +144,94 @@ def test_with_fake_advanced_tagger(self):
         target = write_crazy_file()
         tagger, it, pr = create_auto_tagger(lasla, lower=True)
         out_file = tagger.tag_file(target, it, pr)
+        content = []
         with open(out_file) as f:
-            content = f.read()
-            self.maxDiff = 100000
-            self.assertEqual(
-                content.strip(),
-                """token	lemma	pos	morph
-\\	\\	PUNC	MORPH=empty	\\
-\\	\\	PUNC	MORPH=empty	\\
-\\	\\	PUNC	MORPH=empty	\\
-<	<	PUNC	MORPH=empty	<
-1	lemma0	pos0	Case=Case0|Numb=Numb0|Deg=Deg0|Mood=Mood0|Tense=Tense0|Voice=Voice0|Person=Person0	1
->	>	PUNC	MORPH=empty	>
-[	[	PUNC	MORPH=empty	[
-$	$	PUNC	MORPH=empty	$
-@	@	PUNC	MORPH=empty	@
-$	$	PUNC	MORPH=empty	$
-]	]	PUNC	MORPH=empty	]
-(	(	PUNC	MORPH=empty	(
-v	lemma1	pos1	Case=Case1|Numb=Numb1|Deg=Deg1|Mood=Mood1|Tense=Tense1|Voice=Voice1|Person=Person1	u
-)	)	PUNC	MORPH=empty	)
-\\	\\	PUNC	MORPH=empty	\\
-\\	\\	PUNC	MORPH=empty	\\
-\\	\\	PUNC	MORPH=empty	\\
-§	§	PUNC	MORPH=empty	§
-\\	\\	PUNC	MORPH=empty	\\
-\\	\\	PUNC	MORPH=empty	\\
-<	<	PUNC	MORPH=empty	<
-1	lemma2	pos2	Case=Case2|Numb=Numb2|Deg=Deg2|Mood=Mood2|Tense=Tense2|Voice=Voice2|Person=Person2	1
->	>	PUNC	MORPH=empty	>
-[	[	PUNC	MORPH=empty	[
-$	$	PUNC	MORPH=empty	$
-@	@	PUNC	MORPH=empty	@
-$	$	PUNC	MORPH=empty	$
-]	]	PUNC	MORPH=empty	]
-\\	\\	PUNC	MORPH=empty	\\
-\\	\\	PUNC	MORPH=empty	\\
-§	§	PUNC	MORPH=empty	§
-§	§	PUNC	MORPH=empty	§
-\\	\\	PUNC	MORPH=empty	\\
-[	[	PUNC	MORPH=empty	[
-i	lemma3	pos3	Case=Case3|Numb=Numb3|Deg=Deg3|Mood=Mood3|Tense=Tense3|Voice=Voice3|Person=Person3	i
-]	]	PUNC	MORPH=empty	]
-\\	\\	PUNC	MORPH=empty	\\
-§	§	PUNC	MORPH=empty	§
-en	lemma4	pos4	Case=Case4|Numb=Numb4|Deg=Deg4|Mood=Mood4|Tense=Tense4|Voice=Voice4|Person=Person4	en
-honor	lemma5	pos5	Case=Case5|Numb=Numb5|Deg=Deg5|Mood=Mood5|Tense=Tense5|Voice=Voice5|Person=Person5	honor
-et	lemma6	pos6	Case=Case6|Numb=Numb6|Deg=Deg6|Mood=Mood6|Tense=Tense6|Voice=Voice6|Person=Person6	et
-en	lemma7	pos7	Case=Case7|Numb=Numb7|Deg=Deg7|Mood=Mood7|Tense=Tense7|Voice=Voice7|Person=Person7	en
-bie	lemma8	pos8	Case=Case8|Numb=Numb8|Deg=Deg8|Mood=Mood8|Tense=Tense8|Voice=Voice8|Person=Person8	bie
--ne	ne2	pos9	Case=Case9|Numb=Numb9|Deg=Deg9|Mood=Mood9|Tense=Tense9|Voice=Voice9|Person=Person9	-ne
-et	lemma10	pos10	Case=Case10|Numb=Numb10|Deg=Deg10|Mood=Mood10|Tense=Tense10|Voice=Voice10|Person=Person10	et
-en	lemma11	pos11	Case=Case11|Numb=Numb11|Deg=Deg11|Mood=Mood11|Tense=Tense11|Voice=Voice11|Person=Person11	en
-gra	lemma12	pos12	Case=Case12|Numb=Numb12|Deg=Deg12|Mood=Mood12|Tense=Tense12|Voice=Voice12|Person=Person12	gra
--ne	ne2	pos13	Case=Case13|Numb=Numb13|Deg=Deg13|Mood=Mood13|Tense=Tense13|Voice=Voice13|Person=Person13	-ne
-remembrançe	lemma14	pos14	Case=Case14|Numb=Numb14|Deg=Deg14|Mood=Mood14|Tense=Tense14|Voice=Voice14|Person=Person14	remembrançe
-§	§	PUNC	MORPH=empty	§
-et	lemma15	pos15	Case=Case15|Numb=Numb15|Deg=Deg15|Mood=Mood15|Tense=Tense15|Voice=Voice15|Person=Person15	et
-offerant	lemma16	pos16	Case=Case16|Numb=Numb16|Deg=Deg16|Mood=Mood16|Tense=Tense16|Voice=Voice16|Person=Person16	offerant
-mercé	lemma17	pos17	Case=Case17|Numb=Numb17|Deg=Deg17|Mood=Mood17|Tense=Tense17|Voice=Voice17|Person=Person17	mercé
-,	,	PUNC	MORPH=empty	,
-honor	lemma18	pos18	Case=Case18|Numb=Numb18|Deg=Deg18|Mood=Mood18|Tense=Tense18|Voice=Voice18|Person=Person18	honor
-et	lemma19	pos19	Case=Case19|Numb=Numb19|Deg=Deg19|Mood=Mood19|Tense=Tense19|Voice=Voice19|Person=Person19	et
-celebrançe	lemma20	pos20	Case=Case20|Numb=Numb20|Deg=Deg20|Mood=Mood20|Tense=Tense20|Voice=Voice20|Person=Person20	celebrançe
-§	§	PUNC	MORPH=empty	§""")
+            header = []
+            for line in f:
+                splitted = line.strip().split()
+                if not header:
+                    header = splitted
+                    continue
+                content.append(dict(list(zip(header, splitted))))
+
+        self.assertEqual(
+                content,
+                [{'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '<', 'lemma': '<', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '1', 'lemma': 'lemma0', 'pos': 'pos0',
+                  'morph': 'Case=Case0|Numb=Numb0|Deg=Deg0|Mood=Mood0|Tense=Tense0|Voice=Voice0|Person=Person0'},
+                 {'token': '>', 'lemma': '>', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '[', 'lemma': '[', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '$', 'lemma': '$', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '@', 'lemma': '@', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '$', 'lemma': '$', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': ']', 'lemma': ']', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '(', 'lemma': '(', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': 'v', 'lemma': 'lemma1', 'pos': 'pos1',
+                  'morph': 'Case=Case1|Numb=Numb1|Deg=Deg1|Mood=Mood1|Tense=Tense1|Voice=Voice1|Person=Person1'},
+                 {'token': ')', 'lemma': ')', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '§', 'lemma': '§', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '<', 'lemma': '<', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '1', 'lemma': 'lemma2', 'pos': 'pos2',
+                  'morph': 'Case=Case2|Numb=Numb2|Deg=Deg2|Mood=Mood2|Tense=Tense2|Voice=Voice2|Person=Person2'},
+                 {'token': '>', 'lemma': '>', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '[', 'lemma': '[', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '$', 'lemma': '$', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '@', 'lemma': '@', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '$', 'lemma': '$', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': ']', 'lemma': ']', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '§', 'lemma': '§', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '§', 'lemma': '§', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '[', 'lemma': '[', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': 'i', 'lemma': 'lemma3', 'pos': 'pos3',
+                  'morph': 'Case=Case3|Numb=Numb3|Deg=Deg3|Mood=Mood3|Tense=Tense3|Voice=Voice3|Person=Person3'},
+                 {'token': ']', 'lemma': ']', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '\\', 'lemma': '\\', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': '§', 'lemma': '§', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': 'en', 'lemma': 'lemma4', 'pos': 'pos4',
+                  'morph': 'Case=Case4|Numb=Numb4|Deg=Deg4|Mood=Mood4|Tense=Tense4|Voice=Voice4|Person=Person4'},
+                 {'token': 'honor', 'lemma': 'lemma5', 'pos': 'pos5',
+                  'morph': 'Case=Case5|Numb=Numb5|Deg=Deg5|Mood=Mood5|Tense=Tense5|Voice=Voice5|Person=Person5'},
+                 {'token': 'et', 'lemma': 'lemma6', 'pos': 'pos6',
+                  'morph': 'Case=Case6|Numb=Numb6|Deg=Deg6|Mood=Mood6|Tense=Tense6|Voice=Voice6|Person=Person6'},
+                 {'token': 'en', 'lemma': 'lemma7', 'pos': 'pos7',
+                  'morph': 'Case=Case7|Numb=Numb7|Deg=Deg7|Mood=Mood7|Tense=Tense7|Voice=Voice7|Person=Person7'},
+                 {'token': 'bie', 'lemma': 'lemma8', 'pos': 'pos8',
+                  'morph': 'Case=Case8|Numb=Numb8|Deg=Deg8|Mood=Mood8|Tense=Tense8|Voice=Voice8|Person=Person8'},
+                 {'token': '-ne', 'lemma': 'ne2', 'pos': 'pos9',
+                  'morph': 'Case=Case9|Numb=Numb9|Deg=Deg9|Mood=Mood9|Tense=Tense9|Voice=Voice9|Person=Person9'},
+                 {'token': 'et', 'lemma': 'lemma10', 'pos': 'pos10',
+                  'morph': 'Case=Case10|Numb=Numb10|Deg=Deg10|Mood=Mood10|Tense=Tense10|Voice=Voice10|Person=Person10'},
+                 {'token': 'en', 'lemma': 'lemma11', 'pos': 'pos11',
+                  'morph': 'Case=Case11|Numb=Numb11|Deg=Deg11|Mood=Mood11|Tense=Tense11|Voice=Voice11|Person=Person11'},
+                 {'token': 'gra', 'lemma': 'lemma12', 'pos': 'pos12',
+                  'morph': 'Case=Case12|Numb=Numb12|Deg=Deg12|Mood=Mood12|Tense=Tense12|Voice=Voice12|Person=Person12'},
+                 {'token': '-ne', 'lemma': 'ne2', 'pos': 'pos13',
+                  'morph': 'Case=Case13|Numb=Numb13|Deg=Deg13|Mood=Mood13|Tense=Tense13|Voice=Voice13|Person=Person13'},
+                 {'token': 'remembrançe', 'lemma': 'lemma14', 'pos': 'pos14',
+                  'morph': 'Case=Case14|Numb=Numb14|Deg=Deg14|Mood=Mood14|Tense=Tense14|Voice=Voice14|Person=Person14'},
+                 {'token': '§', 'lemma': '§', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': 'et', 'lemma': 'lemma15', 'pos': 'pos15',
+                  'morph': 'Case=Case15|Numb=Numb15|Deg=Deg15|Mood=Mood15|Tense=Tense15|Voice=Voice15|Person=Person15'},
+                 {'token': 'offerant', 'lemma': 'lemma16', 'pos': 'pos16',
+                  'morph': 'Case=Case16|Numb=Numb16|Deg=Deg16|Mood=Mood16|Tense=Tense16|Voice=Voice16|Person=Person16'},
+                 {'token': 'mercé', 'lemma': 'lemma17', 'pos': 'pos17',
+                  'morph': 'Case=Case17|Numb=Numb17|Deg=Deg17|Mood=Mood17|Tense=Tense17|Voice=Voice17|Person=Person17'},
+                 {'token': ',', 'lemma': ',', 'pos': 'PUNC', 'morph': 'MORPH=empty'},
+                 {'token': 'honor', 'lemma': 'lemma18', 'pos': 'pos18',
+                  'morph': 'Case=Case18|Numb=Numb18|Deg=Deg18|Mood=Mood18|Tense=Tense18|Voice=Voice18|Person=Person18'},
+                 {'token': 'et', 'lemma': 'lemma19', 'pos': 'pos19',
+                  'morph': 'Case=Case19|Numb=Numb19|Deg=Deg19|Mood=Mood19|Tense=Tense19|Voice=Voice19|Person=Person19'},
+                 {'token': 'celebrançe', 'lemma': 'lemma20', 'pos': 'pos20',
+                  'morph': 'Case=Case20|Numb=Numb20|Deg=Deg20|Mood=Mood20|Tense=Tense20|Voice=Voice20|Person=Person20'},
+                 {'token': '§', 'lemma': '§', 'pos': 'PUNC', 'morph': 'MORPH=empty'}])