add .doc and .ppt support (#86)

* add .doc and .ppt support * update lock
su77ungr · May 17, 2023 · 7bab669 · 7bab669
1 parent 8b1453e
commit 7bab669
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 108 deletions.
diff --git a/casalioy/ingest.py b/casalioy/ingest.py
@@ -10,14 +10,14 @@
 from langchain.docstore.document import Document
 from langchain.document_loaders import (
     CSVLoader,
-    Docx2txtLoader,
     OutlookMessageLoader,
     PDFMinerLoader,
     TextLoader,
     UnstructuredEmailLoader,
     UnstructuredEPubLoader,
     UnstructuredHTMLLoader,
     UnstructuredPowerPointLoader,
+    UnstructuredWordDocumentLoader,
 )
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from load_env import chunk_overlap, chunk_size, documents_directory, get_embedding_model, ingest_n_threads, persist_directory
@@ -37,8 +37,10 @@ class Ingester:
         "csv": CSVLoader,
         "epub": UnstructuredEPubLoader,
         "html": UnstructuredHTMLLoader,
-        "docx": Docx2txtLoader,
+        "docx": UnstructuredWordDocumentLoader,
+        "doc": UnstructuredWordDocumentLoader,
         "pptx": UnstructuredPowerPointLoader,
+        "ppt": UnstructuredPowerPointLoader,
         "eml": UnstructuredEmailLoader,
         "msg": OutlookMessageLoader,
     }

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -25,6 +25,7 @@ qdrant-client = "^1.1.7"
 unstructured = "^0.6.6"  # Handle ingestion file formats
 pypandoc-binary = "^1.11"  # doc conversion
 docx2txt = "^0.8"  # Handle docx ingestion file formats
+tabulate = "^0.9.0"  # Also required for docx
 extract-msg = "^0.41.1"  # Handle email file formats
 llama-cpp-python = "^0.1.50"  # 0.1.50 raises an AssertionError / NameError on <5 vic models
 sentence_transformers = "^2.2.2"  # doesn't install torch properly with poetry, but should be better in later versions