Easy RAG

document-parsers/langchain4j-document-parser-apache-tika/pom.xml

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>dev.langchain4j</groupId>
+        <artifactId>langchain4j-parent</artifactId>
+        <version>0.28.0</version>
+        <relativePath>../../langchain4j-parent/pom.xml</relativePath>
+    </parent>
+
+    <artifactId>langchain4j-document-parser-apache-tika</artifactId>
+    <name>LangChain4j :: Document parser :: Apache Tika</name>
+    <packaging>jar</packaging>
+
+    <properties>
+        <apache.tika.version>2.9.1</apache.tika.version>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-core</artifactId>
+            <version>${apache.tika.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-parsers-standard-package</artifactId>
+            <version>${apache.tika.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>

document-parsers/langchain4j-document-parser-apache-tika/src/main/java/dev/langchain4j/data/document/parser/apache/tika/ApacheTikaDocumentParser.java

@@ -0,0 +1,73 @@
+package dev.langchain4j.data.document.parser.apache.tika;
+
+import dev.langchain4j.data.document.Document;
+import dev.langchain4j.data.document.DocumentParser;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+
+import java.io.InputStream;
+
+import static dev.langchain4j.internal.Utils.getOrDefault;
+
+/**
+ * Parses files into {@link Document}s using Apache Tika library, automatically detecting the file format.
+ * This parser supports various file formats, including PDF, DOC, PPT, XLS.
+ * For detailed information on supported formats,
+ * please refer to the <a href="https://tika.apache.org/2.9.1/formats.html">Apache Tika documentation</a>.
+ */
+public class ApacheTikaDocumentParser implements DocumentParser {
+
+    private static final int UNLIMITED = -1;
+
+    private final Parser parser;
+    private final ContentHandler contentHandler;
+    private final Metadata metadata;
+    private final ParseContext parseContext;
+
+    /**
+     * Creates an instance of {@code ApacheTikaDocumentParser} with the default Tika components.
+     * It uses {@link AutoDetectParser}, {@link BodyContentHandler} without write limit,
+     * empty {@link Metadata} and empty {@link ParseContext}.
+     */
+    public ApacheTikaDocumentParser() {
+        this(null, null, null, null);
+    }
+
+    /**
+     * Creates an instance of {@code ApacheTikaDocumentParser} with the provided Tika components.
+     * If some of the components is not provided ({@code null}, the default will be used.
+     * It uses {@link AutoDetectParser}, {@link BodyContentHandler} without write limit,
+     * empty {@link Metadata} and empty {@link ParseContext}.
+     *
+     * @param parser         Tika parser to use. Default: {@link AutoDetectParser}
+     * @param contentHandler Tika content handler. Default: {@link BodyContentHandler} without write limit
+     * @param metadata       Tika metadata. Default: empty {@link Metadata}
+     * @param parseContext   Tika parse context. Default: empty {@link ParseContext}
+     */
+    public ApacheTikaDocumentParser(Parser parser,
+                                    ContentHandler contentHandler,
+                                    Metadata metadata,
+                                    ParseContext parseContext) {
+        this.parser = getOrDefault(parser, new AutoDetectParser());
+        this.contentHandler = getOrDefault(contentHandler, new BodyContentHandler(UNLIMITED));
+        this.metadata = getOrDefault(metadata, new Metadata());
+        this.parseContext = getOrDefault(parseContext, new ParseContext());
+    }
+
+    // TODO allow automatically extract metadata (e.g. creator, last-author, created/modified timestamp, etc)
+
+    @Override
+    public Document parse(InputStream inputStream) {
+        try {
+            parser.parse(inputStream, contentHandler, metadata, parseContext);
+            String text = contentHandler.toString();
+            return Document.from(text);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

document-parsers/langchain4j-document-parser-apache-tika/src/test/java/dev/langchain4j/data/document/parser/apache/tika/ApacheTikaDocumentParserTest.java

@@ -0,0 +1,50 @@
+package dev.langchain4j.data.document.parser.apache.tika;
+
+import dev.langchain4j.data.document.Document;
+import dev.langchain4j.data.document.DocumentParser;
+import org.apache.tika.parser.AutoDetectParser;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.InputStream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class ApacheTikaDocumentParserTest {
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "test-file.doc",
+            "test-file.docx",
+            "test-file.ppt",
+            "test-file.pptx",
+            "test-file.pdf"
+    })
+    void should_parse_doc_ppt_and_pdf_files(String fileName) {
+
+        DocumentParser parser = new ApacheTikaDocumentParser();
+        InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName);
+
+        Document document = parser.parse(inputStream);
+
+        assertThat(document.text()).isEqualToIgnoringWhitespace("test content");
+        assertThat(document.metadata().asMap()).isEmpty();
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "test-file.xls",
+            "test-file.xlsx"
+    })
+    void should_parse_xls_files(String fileName) {
+
+        DocumentParser parser = new ApacheTikaDocumentParser(new AutoDetectParser(), null, null, null);
+        InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName);
+
+        Document document = parser.parse(inputStream);
+
+        assertThat(document.text())
+                .isEqualToIgnoringWhitespace("Sheet1\ntest content\nSheet2\ntest content");
+        assertThat(document.metadata().asMap()).isEmpty();
+    }
+}

langchain4j-bom/pom.xml

@@ -259,6 +259,12 @@
 
             <!-- document parsers -->
 
+            <dependency>
+                <groupId>dev.langchain4j</groupId>
+                <artifactId>langchain4j-document-parser-apache-pdfbox</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>dev.langchain4j</groupId>
                 <artifactId>langchain4j-document-parser-apache-poi</artifactId>
@@ -267,7 +273,7 @@
 
             <dependency>
                 <groupId>dev.langchain4j</groupId>
-                <artifactId>langchain4j-document-parser-apache-pdfbox</artifactId>
+                <artifactId>langchain4j-document-parser-apache-tika</artifactId>
                 <version>${project.version}</version>
             </dependency>
 

langchain4j-easy-rag/pom.xml

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>dev.langchain4j</groupId>
+        <artifactId>langchain4j-parent</artifactId>
+        <version>0.28.0</version>
+        <relativePath>../langchain4j-parent/pom.xml</relativePath>
+    </parent>
+
+    <artifactId>langchain4j-easy-rag</artifactId>
+    <packaging>jar</packaging>
+
+    <name>LangChain4j :: Easy RAG</name>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-document-parser-apache-tika</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-embeddings-all-minilm-l6-v2-q</artifactId>
+            <version>${project.version}</version>
+            <!--            TODO-->
+        </dependency>
+
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-open-ai</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.tinylog</groupId>
+            <artifactId>tinylog-impl</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.tinylog</groupId>
+            <artifactId>slf4j-tinylog</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>