make it working with both master and processor branch of ucto

LanguageMachines · Mar 11, 2019 · 2de4458 · 2de4458
1 parent e07e1b0
commit 2de4458
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 28 deletions.
diff --git a/include/frog/ucto_tokenizer_mod.h b/include/frog/ucto_tokenizer_mod.h
@@ -49,8 +49,8 @@ class UctoTokenizer {
   void setDocID( const std::string& );
   void setTextRedundancy( const std::string& );
   folia::Document *tokenizestring( const std::string& );
+  folia::Document *tokenize_folia( const std::string& );
   folia::Document *tokenize( std::istream& );
-  bool tokenize( folia::Document& );
   std::vector<std::string> tokenize( const std::string&  );
   std::string tokenizeStream( std::istream& );
  private:

diff --git a/src/FrogAPI.cxx b/src/FrogAPI.cxx
@@ -645,26 +645,20 @@ void FrogAPI::FrogServer( Sockets::ServerSocket &conn ){
         if ( options.debugFlag > 5 ){
 	  LOG << "received data [" << result << "]" << endl;
 	}
-	folia::Document doc;
-        try {
-	  doc.readFromString( result );
-        }
-	catch ( std::exception& e ){
-	  LOG << "FoLiaParsing failed:" << endl << e.what() << endl;
-	  throw;
-        }
         LOG << "Processing XML... " << endl;
 	timers.reset();
 	timers.tokTimer.start();
-	tokenizer->tokenize( doc );
+	folia::Document *doc;
+	doc = tokenizer->tokenize_folia( result );
 	timers.tokTimer.stop();
-        FrogDoc( doc );
+        FrogDoc( *doc );
 	if ( options.doXMLout ){
-	  doc.save( outputstream, options.doKanon );
+	  doc->save( outputstream, options.doKanon );
 	}
 	else {
-	  showResults( outputstream, doc );
+	  showResults( outputstream, *doc );
 	}
+	delete doc;
 	//        LOG << "Done Processing XML... " << endl;
       }
       else {
@@ -1489,26 +1483,25 @@ void FrogAPI::FrogFile( const string& infilename,
   if ( xml_in ){
     timers.reset();
     timers.tokTimer.start();
-    folia::Document doc;
+    folia::Document *doc;
     try {
-      doc.readFromFile( infilename );
+      doc = tokenizer->tokenize_folia( infilename );
     }
     catch ( exception &e ){
       LOG << "retrieving FoLiA from '" << infilename << "' failed with exception:" << endl;
       LOG << e.what() << endl;
       throw ( runtime_error( "read failed" ) );
     }
-    tokenizer->setInputXml(true); // THIS IS SILLY, the tokenizer knows it get FoLiA
-    tokenizer->tokenize( doc );
     timers.tokTimer.stop();
-    FrogDoc( doc );
+    FrogDoc( *doc );
     if ( !options.noStdOut ){
-      showResults( os, doc );
+      showResults( os, *doc );
     }
     if ( !xmlOutFile.empty() ){
-      doc.save( xmlOutFile, options.doKanon );
+      doc->save( xmlOutFile, options.doKanon );
       LOG << "resulting FoLiA doc saved in " << xmlOutFile << endl;
     }
+    delete doc;
   }
   else {
     ifstream IN( infilename );

diff --git a/src/ucto_tokenizer_mod.cxx b/src/ucto_tokenizer_mod.cxx
@@ -245,7 +245,24 @@ string UctoTokenizer::tokenizeStream( istream& is ){
   else
     throw runtime_error( "ucto tokenizer not initialized" );
 }
+
+folia::Document *UctoTokenizer::tokenize_folia( const string& buffer ){
+  if ( !tokenizer ){
+    throw runtime_error( "ucto tokenizer not initialized" );
+  }
+  folia::Document *doc = new folia::Document();
+  if ( buffer.find("<?xml " ) == 0 ){
+    doc->readFromString( buffer );
+  }
+  else {
+    doc->readFromFile( buffer );
+  }
+  tokenize( *doc );
+  return doc;
+}
+
 #else
+
 vector<string> UctoTokenizer::tokenize( const string& line ){
   if ( tokenizer ){
     tokenizer->reset();
@@ -264,6 +281,16 @@ string UctoTokenizer::tokenizeStream( istream& is ){
   else
     throw runtime_error( "ucto tokenizer not initialized" );
 }
+
+folia::Document *UctoTokenizer::tokenize_folia( const string& buffer ){
+  if ( !tokenizer ){
+    throw runtime_error( "ucto tokenizer not initialized" );
+  }
+  else {
+    return tokenizer->tokenize_folia( buffer );
+  }
+}
+
 #endif
 
 folia::Document *UctoTokenizer::tokenize( istream& is ){
@@ -281,11 +308,3 @@ folia::Document *UctoTokenizer::tokenizestring( const string& s){
   else
     throw runtime_error( "ucto tokenizer not initialized" );
 }
-
-bool UctoTokenizer::tokenize( folia::Document& doc ){
-  if ( tokenizer )
-    return tokenizer->tokenize( doc );
-  else
-    throw runtime_error( "ucto tokenizer not initialized" );
-
-}