Fix for TIKA-1699 contributed by Sujen Shah

tika-parsers/pom.xml

@@ -232,6 +232,45 @@
       <version>0.7</version>
     </dependency>
 
+	<!--  GROBID Dependencies -->
+	<dependency>
+	  <groupId>org.grobid</groupId>
+  	  <artifactId>grobid-core</artifactId>
+	  <version>0.3.4-SNAPSHOT</version>
+	</dependency>
+	<dependency>
+	  <groupId>directory-naming</groupId>
+	  <artifactId>naming-java</artifactId>
+	  <version>0.8</version>
+	</dependency>
+	<dependency>
+	  <groupId>org.apache.commons</groupId>
+	  <artifactId>commons-lang3</artifactId>
+	  <version>3.0.1</version>
+	</dependency>		
+	<dependency>
+	  <groupId>org.chasen</groupId>
+	  <artifactId>crfpp</artifactId>
+	  <version>1.0.2</version>
+	  <scope>compile</scope>
+	</dependency>
+	<dependency>
+      <groupId>fr.limsi.wapiti</groupId>
+      <artifactId>wapiti</artifactId>
+      <version>1.5.0</version>
+    </dependency>
+    <dependency>
+	  <groupId>com.cybozu</groupId>
+	  <artifactId>language-detection</artifactId>
+	  <version>09-13-2011</version>
+	</dependency>
+    <dependency>
+	  <groupId>net.arnx</groupId>
+	  <artifactId>jsonic</artifactId>
+	  <version>1.3.5</version>
+	</dependency>
+
+
     <!-- Provided dependencies -->
     <dependency>
       <groupId>org.xerial</groupId>

tika-parsers/src/main/java/org/apache/tika/parser/journal/GrobidConfig.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.journal;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+public class GrobidConfig {
+
+  public static final String GROBID_PREFIX = "grobid:";
+  public static final String HEADER_METADATA_PREFIX = "header_";
+
+  private String grobidHome; 
+  private String grobidProperties; 
+
+  public GrobidConfig() {
+    init(this.getClass().getResourceAsStream("GrobidExtractor.properties"));
+  }
+
+  private void init(InputStream in) {
+    if (in == null) {
+      return;
+    }
+
+    Properties props = new Properties();
+    try {
+      props.load(in);
+    } catch (IOException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    } finally {
+      try {
+        in.close();
+      } catch (IOException e) {
+        // TODO Auto-generated catch block
+        e.printStackTrace();
+      }
+    }
+
+    setGrobidHome(props.getProperty("grobid.home", getGrobidHome()));
+    setGrobidProperties(props.getProperty("grobid.properties", getGrobidProperties()));
+  }
+
+  public String getGrobidHome() {
+    return grobidHome;
+  }
+
+  public void setGrobidHome(String grobidHome) {
+    this.grobidHome = grobidHome;
+  }
+
+  public String getGrobidProperties() {
+    return grobidProperties;
+  }
+
+  public void setGrobidProperties(String grobidProperties) {
+    this.grobidProperties = grobidProperties;
+  }
+}

tika-parsers/src/main/java/org/apache/tika/parser/journal/GrobidHeaderMetadata.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.journal;
+
+import java.beans.BeanInfo;
+import java.beans.IntrospectionException;
+import java.beans.Introspector;
+import java.beans.PropertyDescriptor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.grobid.core.data.BiblioItem;
+
+public class GrobidHeaderMetadata {
+
+
+  private Map<String, String> headerMetadata;
+
+  public void generateHeaderMetada(BiblioItem resHeader) {
+    headerMetadata = new HashMap<String, String>();
+    try {
+      BeanInfo info = Introspector.getBeanInfo(BiblioItem.class);
+
+      for (PropertyDescriptor pd : info.getPropertyDescriptors()) {
+        Method m = pd.getReadMethod();
+        if( m!= null) {
+          Object value = m.invoke(resHeader);
+          if(value != null){
+            headerMetadata.put(GrobidConfig.HEADER_METADATA_PREFIX +  m.getName().replace("get", ""), "" +value);
+          }
+
+        }
+      }
+    } catch (IntrospectionException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+  }
+
+  public Map<String, String> getHeaderMetadata() {
+    return headerMetadata;
+  }
+
+  public void setHeaderMetadata(Map<String, String> headerMetadata) {
+    this.headerMetadata = headerMetadata;
+  }
+}

tika-parsers/src/main/java/org/apache/tika/parser/journal/GrobidParser.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.journal;
+
+import java.util.Map.Entry;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.grobid.core.data.BiblioItem;
+import org.grobid.core.engines.Engine;
+import org.grobid.core.factory.GrobidFactory;
+import org.grobid.core.mock.MockContext;
+import org.grobid.core.utilities.GrobidProperties;
+import org.xml.sax.ContentHandler;
+
+public class GrobidParser {
+
+  public GrobidParser(){
+
+  }
+
+  public void parse(String filePath, ContentHandler handler, Metadata metadata, ParseContext context) {
+    GrobidConfig gConfig = new GrobidConfig();
+    try {
+      MockContext.setInitialContext(gConfig.getGrobidHome(), gConfig.getGrobidProperties());
+      GrobidProperties.getInstance();
+
+      Engine engine = GrobidFactory.getInstance().createEngine();
+      BiblioItem resHeader = new BiblioItem();
+      engine.processHeader(filePath, false, resHeader);
+      GrobidHeaderMetadata gheaderMetada = new GrobidHeaderMetadata();
+      gheaderMetada.generateHeaderMetada(resHeader);
+      populateTikaMetadata(gheaderMetada, metadata);
+    } catch (Exception e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+  }
+
+  private void populateTikaMetadata(GrobidHeaderMetadata gheaderMetada, Metadata metadata) {
+    for(Entry<String, String> pair: gheaderMetada.getHeaderMetadata().entrySet()) {
+      metadata.add(GrobidConfig.GROBID_PREFIX + pair.getKey(), pair.getValue());
+    }
+  }
+
+}

tika-parsers/src/main/java/org/apache/tika/parser/journal/JournalParser.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.journal;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.pdf.PDFParser;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+public class JournalParser extends AbstractParser{
+
+  /**
+   * Generated serial ID
+   */
+  private static final long serialVersionUID = 4664255544154296438L;
+
+
+  private static final MediaType TYPE =
+      MediaType.application("pdf");
+
+  private static final Set<MediaType> SUPPORTED_TYPES =
+      Collections.singleton(TYPE);
+
+  public Set<MediaType> getSupportedTypes(ParseContext context) {
+    return SUPPORTED_TYPES;
+  }
+
+  public void parse(
+      InputStream stream, ContentHandler handler,
+      Metadata metadata, ParseContext context)
+          throws IOException, SAXException, TikaException {
+    TikaInputStream tis = TikaInputStream.get(stream, new TemporaryResources());
+    File tmpFile = tis.getFile();
+
+    GrobidParser grobidParser = new GrobidParser();
+    grobidParser.parse(tmpFile.getAbsolutePath(), handler, metadata, context);
+
+    PDFParser parser = new PDFParser();
+    parser.parse(new FileInputStream(tmpFile), handler, metadata, context);
+  }
+}

tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser

@@ -66,3 +66,4 @@ org.apache.tika.parser.isatab.ISArchiveParser
 org.apache.tika.parser.geoinfo.GeographicInformationParser
 org.apache.tika.parser.geo.topic.GeoParser
 org.apache.tika.parser.external.CompositeExternalParser
+org.apache.tika.parser.journal.JournalParser

tika-parsers/src/main/resources/org/apache/tika/parser/journal/GrobidExtractor.properties

@@ -0,0 +1,17 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+grobid.home=/path/to/grobid-home
+grobid.properties=/path/to/grobid-home/config/grobid.properties

tika-parsers/src/test/java/org/apache/tika/parser/journal/JournalParserTest.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.journal;
+
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.BodyContentHandler;
+import org.junit.Test;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+public class JournalParserTest{
+
+  @Test
+  public void testJournalParser(){
+    String path = "/test-documents/testJournalParser.pdf";
+    ContentHandler handler = new BodyContentHandler();
+    Metadata metadata = new Metadata();
+    InputStream stream = JournalParserTest.class.getResourceAsStream(path);
+
+    JournalParser jParser = new JournalParser();
+    try {
+      jParser.parse(stream, handler, metadata, new ParseContext());
+    } catch (IOException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    } catch (SAXException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    } catch (TikaException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+
+    assertNotNull(metadata.get("grobid:header_Title"));
+  }
+}