diff --git a/bootstraprag/templates/evaluations/ragas_evals/.env b/bootstraprag/templates/evaluations/ragas_evals/.env index 66c3d61..3649c94 100644 --- a/bootstraprag/templates/evaluations/ragas_evals/.env +++ b/bootstraprag/templates/evaluations/ragas_evals/.env @@ -1,12 +1,12 @@ input_dir='data' collection_name='topic_node_parser' -qdrant_url='http://localhost:6333' +qdrant_url='http://localhost:6333/' qdrant_api_key='th3s3cr3tk3y' llm_url='http://localhost:11434' llm_model='llama3.2:latest' embed_model_name='nomic-embed-text:latest' sparse_model='Qdrant/bm42-all-minilm-l6-v2-attentions' -OPENAI_API_KEY= +OPENAI_API_KEY='sk-proj-' retrieve_top_k=5 enable_postprocessing_method=false postprocessing_method='llm_reranker' diff --git a/bootstraprag/templates/evaluations/ragas_evals/llamaindex/llamaindex_ragas.py b/bootstraprag/templates/evaluations/ragas_evals/llamaindex/llamaindex_ragas.py index 81a43f5..3e3a159 100644 --- a/bootstraprag/templates/evaluations/ragas_evals/llamaindex/llamaindex_ragas.py +++ b/bootstraprag/templates/evaluations/ragas_evals/llamaindex/llamaindex_ragas.py @@ -1,6 +1,6 @@ import os import logging -from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings +from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings, StorageContext from llama_index.vector_stores.qdrant import QdrantVectorStore from llama_index.core.postprocessor import LLMRerank, SentenceTransformerRerank, LongContextReorder from llama_index.llms.openai import OpenAI @@ -30,12 +30,16 @@ def __init__(self, input_dir="../data", model_name=os.environ.get("llm_model"), logger.info( f"Initializing LlamaIndexEvaluator with input_dir={input_dir}, model_name={model_name}, base_url={base_url}") + Settings.llm = Ollama(model=os.environ.get('llm_model'), base_url=base_url) + Settings.embed_model = OllamaEmbedding(model_name=os.environ.get('embed_model_name'), base_url=base_url) + self.input_dir = input_dir self.model_name = model_name self.base_url = base_url self.documents = self.load_documents() + self.index_data(documents=self.documents) self.top_k = int(os.environ.get('retrieve_top_k')) - Settings.llm = Ollama(model=self.model_name, base_url=self.base_url) + logger.info(f"Loading dataset from {os.environ.get('gold_dataset_file')}") self.dataset = pd.read_json(os.path.join(input_dir, os.environ.get('gold_dataset_file'))) self.query_engine = self.build_query_engine(postprocessing_method=os.environ.get('postprocessing_method')) @@ -48,6 +52,17 @@ def load_documents(self): logger.info(f"Loaded {len(documents)} documents") return documents + def index_data(self, documents): + logger.info("Indexing data in Qdrant") + # Initialize Qdrant client + qdrant_cli = qdrant_client.QdrantClient(url=os.environ.get("qdrant_url"), + api_key=os.environ.get("qdrant_api_key")) + qdrant_vector_store = QdrantVectorStore(collection_name=os.environ.get("collection_name"), client=qdrant_cli) + storage_context = StorageContext.from_defaults(vector_store=qdrant_vector_store) + if not qdrant_cli.collection_exists(collection_name=os.environ.get("collection_name")): + VectorStoreIndex.from_documents(storage_context=storage_context, documents=documents, show_progress=True) + logger.info("Indexing data in Qdrant finished") + def build_query_engine(self, postprocessing_method: str): logger.info("Initializing Qdrant client") # Initialize Qdrant client @@ -71,16 +86,18 @@ def build_query_engine(self, postprocessing_method: str): if os.environ.get('enable_postprocessing_method') == 'true' and postprocessing_method == 'llm_reranker': reranker = LLMRerank(llm=Settings.llm, choice_batch_size=self.top_k) query_engine = vector_index.as_query_engine(top_k=self.top_k, node_postprocessors=[reranker]) - elif os.environ.get('enable_postprocessing_method') == 'true' and postprocessing_method == 'sentence_transformer_rerank': + elif os.environ.get( + 'enable_postprocessing_method') == 'true' and postprocessing_method == 'sentence_transformer_rerank': reranker = SentenceTransformerRerank( model="cross-encoder/ms-marco-MiniLM-L-2-v2", top_n=self.top_k ) query_engine = vector_index.as_query_engine(top_k=self.top_k, node_postprocessors=[reranker]) - elif os.environ.get('enable_postprocessing_method') == 'true' and postprocessing_method == 'long_context_reorder': + elif os.environ.get( + 'enable_postprocessing_method') == 'true' and postprocessing_method == 'long_context_reorder': reorder = LongContextReorder() query_engine = vector_index.as_query_engine(top_k=self.top_k, node_postprocessors=[reorder]) - - query_engine = vector_index.as_query_engine(similarity_top_k=self.top_k, llm=Settings.llm) + else: + query_engine = vector_index.as_query_engine(similarity_top_k=self.top_k, llm=Settings.llm) logger.info(f"Query engine built successfully: {query_engine}") return query_engine diff --git a/bootstraprag/templates/llamaindex/rag_with_controllable_agents_with_observability/docker-compose-langfuse.yml b/bootstraprag/templates/llamaindex/rag_with_controllable_agents_with_observability/docker-compose-langfuse.yml index b6b36be..2363657 100644 --- a/bootstraprag/templates/llamaindex/rag_with_controllable_agents_with_observability/docker-compose-langfuse.yml +++ b/bootstraprag/templates/llamaindex/rag_with_controllable_agents_with_observability/docker-compose-langfuse.yml @@ -2,11 +2,10 @@ version: '3.8' services: postgres: - image: postgres:13 + image: postgres:latest container_name: postgres environment: - POSTGRES_USER: admin - POSTGRES_PASSWORD: admin@123 + POSTGRES_PASSWORD: postgres POSTGRES_DB: langfuse volumes: - postgres_data:/var/lib/postgresql/data @@ -17,7 +16,7 @@ services: image: langfuse/langfuse:latest container_name: langfuse environment: - DATABASE_URL: postgresql://hello:mysecretpassword@postgres:5432/langfuse + DATABASE_URL: postgresql://postgres:postgres@postgres:5432/langfuse NEXTAUTH_URL: http://localhost:3000 NEXTAUTH_SECRET: mysecret SALT: mysalt diff --git a/bootstraprag/templates/llamaindex/rag_with_hyde/.env b/bootstraprag/templates/llamaindex/rag_with_hyde/.env index 595f232..f9fa2a9 100644 --- a/bootstraprag/templates/llamaindex/rag_with_hyde/.env +++ b/bootstraprag/templates/llamaindex/rag_with_hyde/.env @@ -1,6 +1,6 @@ DB_URL='http://localhost:6333' DB_API_KEY='th3s3cr3tk3y' -COLLECTION_NAME='YOUR_COLLECTION' +COLLECTION_NAME='HYDE_COLLECTION' OPENAI_API_KEY='' OPENAI_EMBED_MODEL='' diff --git a/bootstraprag/templates/llamaindex/rag_with_hyde/base_rag.py b/bootstraprag/templates/llamaindex/rag_with_hyde/base_rag.py index ce32821..097f3bb 100644 --- a/bootstraprag/templates/llamaindex/rag_with_hyde/base_rag.py +++ b/bootstraprag/templates/llamaindex/rag_with_hyde/base_rag.py @@ -65,7 +65,7 @@ def __init__(self, data_path: str, chunk_size: int = 512, chunk_overlap: int = 2 Settings.llm = llm Settings.transformations = [self.text_parser] - self.rag_evaluator = RAGEvaluator + self.rag_evaluator = RAGEvaluator() self.text_chunks = [] self.doc_ids = [] diff --git a/bootstraprag/templates/llamaindex/rag_with_hyde_with_observability/docker-compose-langfuse.yml b/bootstraprag/templates/llamaindex/rag_with_hyde_with_observability/docker-compose-langfuse.yml index b6b36be..2363657 100644 --- a/bootstraprag/templates/llamaindex/rag_with_hyde_with_observability/docker-compose-langfuse.yml +++ b/bootstraprag/templates/llamaindex/rag_with_hyde_with_observability/docker-compose-langfuse.yml @@ -2,11 +2,10 @@ version: '3.8' services: postgres: - image: postgres:13 + image: postgres:latest container_name: postgres environment: - POSTGRES_USER: admin - POSTGRES_PASSWORD: admin@123 + POSTGRES_PASSWORD: postgres POSTGRES_DB: langfuse volumes: - postgres_data:/var/lib/postgresql/data @@ -17,7 +16,7 @@ services: image: langfuse/langfuse:latest container_name: langfuse environment: - DATABASE_URL: postgresql://hello:mysecretpassword@postgres:5432/langfuse + DATABASE_URL: postgresql://postgres:postgres@postgres:5432/langfuse NEXTAUTH_URL: http://localhost:3000 NEXTAUTH_SECRET: mysecret SALT: mysalt diff --git a/bootstraprag/templates/llamaindex/rag_with_mlflow/__init__.py b/bootstraprag/templates/llamaindex/rag_with_mlflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstraprag/templates/llamaindex/rag_with_mlflow/readme.md b/bootstraprag/templates/llamaindex/rag_with_mlflow/readme.md new file mode 100644 index 0000000..1897d0e --- /dev/null +++ b/bootstraprag/templates/llamaindex/rag_with_mlflow/readme.md @@ -0,0 +1 @@ +## In progress \ No newline at end of file diff --git a/bootstraprag/templates/llamaindex/rag_with_react_with_observability/docker-compose-langfuse.yml b/bootstraprag/templates/llamaindex/rag_with_react_with_observability/docker-compose-langfuse.yml index b6b36be..2363657 100644 --- a/bootstraprag/templates/llamaindex/rag_with_react_with_observability/docker-compose-langfuse.yml +++ b/bootstraprag/templates/llamaindex/rag_with_react_with_observability/docker-compose-langfuse.yml @@ -2,11 +2,10 @@ version: '3.8' services: postgres: - image: postgres:13 + image: postgres:latest container_name: postgres environment: - POSTGRES_USER: admin - POSTGRES_PASSWORD: admin@123 + POSTGRES_PASSWORD: postgres POSTGRES_DB: langfuse volumes: - postgres_data:/var/lib/postgresql/data @@ -17,7 +16,7 @@ services: image: langfuse/langfuse:latest container_name: langfuse environment: - DATABASE_URL: postgresql://hello:mysecretpassword@postgres:5432/langfuse + DATABASE_URL: postgresql://postgres:postgres@postgres:5432/langfuse NEXTAUTH_URL: http://localhost:3000 NEXTAUTH_SECRET: mysecret SALT: mysalt diff --git a/bootstraprag/templates/llamaindex/rag_with_self_correction_with_observability/docker-compose-langfuse.yml b/bootstraprag/templates/llamaindex/rag_with_self_correction_with_observability/docker-compose-langfuse.yml index b6b36be..2363657 100644 --- a/bootstraprag/templates/llamaindex/rag_with_self_correction_with_observability/docker-compose-langfuse.yml +++ b/bootstraprag/templates/llamaindex/rag_with_self_correction_with_observability/docker-compose-langfuse.yml @@ -2,11 +2,10 @@ version: '3.8' services: postgres: - image: postgres:13 + image: postgres:latest container_name: postgres environment: - POSTGRES_USER: admin - POSTGRES_PASSWORD: admin@123 + POSTGRES_PASSWORD: postgres POSTGRES_DB: langfuse volumes: - postgres_data:/var/lib/postgresql/data @@ -17,7 +16,7 @@ services: image: langfuse/langfuse:latest container_name: langfuse environment: - DATABASE_URL: postgresql://hello:mysecretpassword@postgres:5432/langfuse + DATABASE_URL: postgresql://postgres:postgres@postgres:5432/langfuse NEXTAUTH_URL: http://localhost:3000 NEXTAUTH_SECRET: mysecret SALT: mysalt diff --git a/bootstraprag/templates/llamaindex/rag_with_vision/__init__.py b/bootstraprag/templates/llamaindex/rag_with_vision/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstraprag/templates/llamaindex/rag_with_vision/readme.md b/bootstraprag/templates/llamaindex/rag_with_vision/readme.md new file mode 100644 index 0000000..1897d0e --- /dev/null +++ b/bootstraprag/templates/llamaindex/rag_with_vision/readme.md @@ -0,0 +1 @@ +## In progress \ No newline at end of file diff --git a/bootstraprag/templates/llamaindex/simple_rag_with_observability/docker-compose-langfuse.yml b/bootstraprag/templates/llamaindex/simple_rag_with_observability/docker-compose-langfuse.yml new file mode 100644 index 0000000..2363657 --- /dev/null +++ b/bootstraprag/templates/llamaindex/simple_rag_with_observability/docker-compose-langfuse.yml @@ -0,0 +1,36 @@ +version: '3.8' + +services: + postgres: + image: postgres:latest + container_name: postgres + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: langfuse + volumes: + - postgres_data:/var/lib/postgresql/data + networks: + - langfuse-network + + langfuse: + image: langfuse/langfuse:latest + container_name: langfuse + environment: + DATABASE_URL: postgresql://postgres:postgres@postgres:5432/langfuse + NEXTAUTH_URL: http://localhost:3000 + NEXTAUTH_SECRET: mysecret + SALT: mysalt + ENCRYPTION_KEY: 98637d42c277ef10b8a324e25d492daa8eee8f769574124ba25132f71481f183 + ports: + - "3000:3000" + depends_on: + - postgres + networks: + - langfuse-network + +volumes: + postgres_data: + +networks: + langfuse-network: + driver: bridge diff --git a/bootstraprag/templates/llamaindex/simple_rag_with_observability/requirements.txt b/bootstraprag/templates/llamaindex/simple_rag_with_observability/requirements.txt index 0525aa2..6f21eb7 100644 --- a/bootstraprag/templates/llamaindex/simple_rag_with_observability/requirements.txt +++ b/bootstraprag/templates/llamaindex/simple_rag_with_observability/requirements.txt @@ -1,13 +1,13 @@ python-dotenv==1.0.1 -llama-index==0.11.7 -llama-index-llms-openai==0.2.3 -llama-index-llms-ollama==0.3.1 -llama-index-embeddings-openai==0.2.4 -llama-index-embeddings-ollama==0.3.0 -llama-index-vector-stores-qdrant==0.3.0 -qdrant-client==1.11.1 -arize-phoenix==4.33.1 -llama-index-callbacks-arize-phoenix==0.2.1 -pydantic==2.9.0 -litserve==0.2.2 -deepeval==1.3.2 \ No newline at end of file +llama-index==0.11.19 +llama-index-llms-openai==0.2.16 +llama-index-llms-ollama==0.3.4 +llama-index-embeddings-openai==0.2.5 +llama-index-embeddings-ollama==0.3.1 +llama-index-vector-stores-qdrant==0.3.2 +qdrant-client==1.12.0 +pydantic==2.9.2 +litserve==0.2.3 +deepeval==1.4.4 +# observability +langfuse==2.52.2 \ No newline at end of file diff --git a/bootstraprag/templates/llamaindex/simple_rag_with_observability/simple_rag.py b/bootstraprag/templates/llamaindex/simple_rag_with_observability/simple_rag.py index 066c11e..6bea20e 100644 --- a/bootstraprag/templates/llamaindex/simple_rag_with_observability/simple_rag.py +++ b/bootstraprag/templates/llamaindex/simple_rag_with_observability/simple_rag.py @@ -9,14 +9,13 @@ from llama_index.core.agent import ReActAgent from llama_index.llms.ollama import Ollama from llama_index.core.base.response.schema import Response, StreamingResponse, AsyncStreamingResponse, PydanticResponse +from langfuse.llama_index import LlamaIndexInstrumentor from rag_evaluator import RAGEvaluator from dotenv import load_dotenv, find_dotenv from typing import Union import qdrant_client import logging import os -import llama_index -import phoenix as px _ = load_dotenv(find_dotenv()) @@ -24,8 +23,8 @@ logger = logging.getLogger(__name__) # instrumenting observability -session = px.launch_app() -llama_index.core.set_global_handler("arize_phoenix") +instrumentor = LlamaIndexInstrumentor() +instrumentor.start() class SimpleRAG: diff --git a/setup.py b/setup.py index 37861bd..6183719 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='bootstrap-rag', - version='0.0.11', + version='0.0.12', long_description=long_description, long_description_content_type="text/markdown", packages=find_packages(),