*) using Apache POI library to parse Word documents now

*) removed tm-extractors library (can be found at http://www.textmining.org/ if necessary again) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6193 6c8d7289-2bf4-0310-a012-ef5d649a1542
yacy · Jul 10, 2009 · f242e7d · f242e7d
1 parent caedd72
commit f242e7d
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 11 deletions.
diff --git a/lib/tm-extractors-1.0.jar b/lib/tm-extractors-1.0.jar
diff --git a/source/de/anomic/document/parser/docParser.java b/source/de/anomic/document/parser/docParser.java
@@ -31,14 +31,13 @@
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.util.HashMap;
-import org.textmining.extraction.TextExtractor;
-import org.textmining.extraction.word.WordTextExtractorFactory;
 
 import de.anomic.document.AbstractParser;
 import de.anomic.document.Idiom;
 import de.anomic.document.ParserException;
 import de.anomic.document.Document;
 import de.anomic.yacy.yacyURL;
+import org.apache.poi.hwpf.extractor.WordExtractor;
 
 public class docParser extends AbstractParser implements Idiom {
 
@@ -65,27 +64,33 @@ public docParser() {
 	}
 
 	public Document parse(final yacyURL location, final String mimeType, final String charset, final InputStream source) throws ParserException, InterruptedException {
-       final WordTextExtractorFactory extractorFactory = new WordTextExtractorFactory();
-       TextExtractor extractor = null;
+
+        final WordExtractor extractor;
+
         try {
-            extractor = extractorFactory.textExtractor(source);
-        } catch (Exception e) {
+            extractor = new WordExtractor(source);
+        } catch (IOException e) {
             throw new ParserException("error in docParser, WordTextExtractorFactory: " + e.getMessage(), location);
         }
-		String contents = null;
+
+		StringBuilder contents = new StringBuilder();
         try {
-            contents = extractor.getText().trim();
-        } catch (IOException e) {
+            contents.append(extractor.getText().trim());
+            contents.append(extractor.getHeaderText());
+            contents.append(extractor.getFooterText());
+        } catch (Exception e) {
             throw new ParserException("error in docParser, getText: " + e.getMessage(), location);
         }
-	    String title = contents.replaceAll("\r"," ").replaceAll("\n"," ").replaceAll("\t"," ").trim();
+	    String title = (contents.length() > 240) ? contents.substring(0,240) : contents.toString().trim();
+        title.replaceAll("\r"," ").replaceAll("\n"," ").replaceAll("\t"," ").trim();
 	    if (title.length() > 80) title = title.substring(0, 80);
 	    int l = title.length();
 	    while (true) {
 	        title = title.replaceAll("  ", " ");
 	        if (title.length() == l) break;
 	        l = title.length();
 	    }
+
         Document theDoc;
         try {
             theDoc = new Document(
@@ -98,7 +103,7 @@ public Document parse(final yacyURL location, final String mimeType, final Strin
                       "", // TODO: AUTHOR
                       null,
                       null,
-                      contents.getBytes("UTF-8"),
+                      contents.toString().getBytes("UTF-8"),
                       null,
                       null);
         } catch (UnsupportedEncodingException e) {