-
Notifications
You must be signed in to change notification settings - Fork 0
/
ingest.py
26 lines (20 loc) · 1.01 KB
/
ingest.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import PyPDFLoader, DirectoryLoader
from langchain.embeddings import HuggingFaceEmbeddings # can also import sentence transformers
from langchain.vectorstores import FAISS
# path where medical doc is present
DATA_PATH = "data/"
# path to store vector db in
DB_FAISS_PATH = "vectorstores/db_faiss"
# creating a vector database
def create_vector_db():
# load the pdf present in the given path
loader = DirectoryLoader(DATA_PATH, glob="*.pdf", loader_cls=PyPDFLoader)
documents = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 50)
texts = text_splitter.split_documents(documents)
embeddings = HuggingFaceEmbeddings(model_name = 'sentence-transformers/all-MiniLM-L6-v2', model_kwargs = {'device': 'cpu'})
db = FAISS.from_documents(texts, embeddings)
db.save_local(DB_FAISS_PATH)
if __name__ == '__main__':
create_vector_db()