fixed test assignment to Sentence

LanguageMachines · Apr 15, 2019 · 14cd4c5 · 14cd4c5
1 parent da1aed1
commit 14cd4c5
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 8 deletions.
diff --git a/src/FrogAPI.cxx b/src/FrogAPI.cxx
@@ -632,10 +632,12 @@ folia::FoliaElement *FrogAPI::append_to_folia( folia::FoliaElement *root,
     }
     folia::LangAnnotation *la = new folia::LangAnnotation( args, root->doc() );
     s->append( la );
-    string text = fd.sentence(); // get tokenized, space separated, sentence.
-    text = TiCC::trim( text );
-    if ( !text.empty() ){
-      s->settext( text );
+    if ( options.textredundancy == "full" ){
+      string text = fd.sentence(false); // get detokenized sentence.
+      text = TiCC::trim( text );
+      if ( !text.empty() ){
+	s->settext( s->str(options.outputclass), options.outputclass );
+      }
     }
   }
   else {
@@ -1595,7 +1597,7 @@ void FrogAPI::run_folia_processor( const string& infilename,
   if ( xmlOutFile.empty() ){
     options.noStdOut = false;
   }
-  folia::TextProcessor proc( infilename );
+  folia::TextEngine proc( infilename );
   if ( !options.doTok ){
     proc.declare( folia::AnnotationType::TOKEN, "passthru",
 		  "annotator='ucto', annotatortype='auto', datetime='now()'" );

diff --git a/src/ucto_tokenizer_mod.cxx b/src/ucto_tokenizer_mod.cxx
@@ -427,8 +427,5 @@ vector<folia::Word*> UctoTokenizer::add_words( folia::Sentence* s,
     }
     wv.push_back( w );
   }
-  if ( textredundancy == "full" ){
-    s->settext( s->str(textclass), textclass );
-  }
   return wv;
 }