Merge pull request #327 from UnB-KnEDLe/highlight_patch

Normalize text in json personal acts

dodfminer/__version__.py

@@ -1,4 +1,4 @@
-version_info = (1, 4, 5)
+version_info = (1, 4, 6)
 # format:
 # ('dodf_major', 'dodf_minor', 'dodf_patch')
 

dodfminer/extract/polished/acts/base.py

@@ -6,6 +6,7 @@
 
 import re
 import json
+import unicodedata
 import pandas as pd
 
 from dodfminer.extract.polished.backend.regex import ActRegex
@@ -204,9 +205,8 @@ def highlight_dataframe(self):
             return
         self._data_frame = []
         for IOB, text in zip(self._preds, self._acts_str):
-            ent_dict = {
-                'text': '',
-            } 
+            ent_dict = dict()
+            ent_dict['titulo'] = None
             ent_dict['text'] = ""
 
             text_split = self._split_sentence(text) + ["O"]
@@ -295,6 +295,8 @@ def read_json(self, file_name):
                     txt = re.sub('<[^<]+?>', ' ', txt).replace('&nbsp', ' ')
                     all_txt.append(txt)
         self._text = ''.join(all_txt)
+        self._text = unicodedata.normalize('NFKD', self._text).encode(
+            'ascii', 'ignore').decode('utf8')
 
     def read_txt(self, file_name):
         """Reads a .txt file of a DODF.

dodfminer/extract/polished/acts/base_contratos.py

@@ -124,13 +124,9 @@ def highlight_dataframe(self):
     if len(self.atos_encontrados) == 0:
       return
     self.data_frame = []
-    for IOB, text, numdodf, titulo in zip(self.predicted, self.atos_encontrados['texto'], self.atos_encontrados['numero_dodf'], self.atos_encontrados['titulo']):
-      ent_dict = {
-        'numero_dodf': '',
-        'titulo': '',
-        'text': '',
-      } 
-      ent_dict['numero_dodf'] = numdodf
+    for IOB, text, _, titulo in zip(self.predicted, self.atos_encontrados['texto'], self.atos_encontrados['numero_dodf'], self.atos_encontrados['titulo']):
+      ent_dict = dict() 
+      # ent_dict['numero_dodf'] = numdodf
       ent_dict['titulo'] = titulo
       ent_dict['text'] = ""