diff --git a/evaluator/.env.example b/evaluator/.env.example index 03dbe82..b8a5b83 100644 --- a/evaluator/.env.example +++ b/evaluator/.env.example @@ -1,4 +1,5 @@ MILVUS_ADDR=http://localhost:19530 +LLM_TYPE=model_garden MODEL_GARDEN_URL= MODEL_GARDEN_MODEL= EMBEDDING_URL= diff --git a/evaluator/dataset.csv b/evaluator/dataset.csv index 168c778..3f769be 100644 --- a/evaluator/dataset.csv +++ b/evaluator/dataset.csv @@ -1,4 +1,4 @@ user_input,reference_contexts,reference,persona_name,query_style,query_length,synthesizer_name -"According to the provided text, what was the purpose of building ‘BARITO’?","['H Ho ow w w we e b bu ui il lt t ‘ ‘B BA AR RI IT TO O’ ’ t to o e en nh ha an nc ce e l lo og gg gi in ng g\nBuilding a logging infrastructure commensurate to GO-JEK’s scale\n5 MIN READ \xa0| \xa0 DEC 08, 2018 Share: \nBB yy TT aa r r aa BB aa s s kk aa r r aa\nImagine coming home from a tiring day of work and your white-fenced house is\nsuddenly pink. You’d have questions. Many of them. Did I arrive at the correct\nhouse? Who did this? When did this happen? For what? And why pink? You’d\nlook around frantically for clues. Did any of your neighbours see anything? You’d\nHow we built ‘BARITO’ to enhance logging - 5 min read https://www.gojek.io/blog/how-we-built-barito-to-enhance-logging\n1 of 10 20/11/25, 10.27']",‘BARITO’ was built to enhance logging.,DevOps Engineer - GO-JEK,PERFECT_GRAMMAR,MEDIUM,single_hop_specific_query_synthesizer -"logging infrastructure at GO-JEK, like, why is it so important and what problems did they have with the old ELK stack?","['<1-hop>\n\nwant answers. So imagine how glad you’d be once you realized you’ve installed\nsecurity cameras in front of your house not long ago. All you have to do now is\ntake a look at the log and get all the answers.\nNow, what about complex applications and infrastructures systems? That’s\nthousands of white fences turning into every colour imaginable. How can you\nensure security cams record everything? At this point you’d agree that ll o o g g g g ii n n g g is\none of the most critical parts of any system. It gives us the visibility on how a\nsystem is doing and the ability to identify what went wrong when something\ngoes haywire.\nWhat about GO-JEK? Well, the absence of logging infrastructure brings us\nnothing but havoc. We’ve been through more than our fair and we critically need\nto improve in some areas:\nMean Time T o Respond (MTTR)\nAt GO-JEK, we need to improve our MTTR. We have millions of bookings each\nday. If our MTTR is high, can you imagine the number of drivers missing potential\norders? The number of people who may now be late for their appointments with\neach additional minute we take to solve an issue? Without proper logging, it’s\nakin to finding a needle in a haystack — because you have to check across\nthousands of machines in the faint hope that you might just stumble across the\nproblem.\nELK scale\nAt GO-JEK’s scale, the conventional ELK stack (without sharding) we used did\nnot scale for the quantity of logs. Each team has their own ELK cluster, but\nmaintaining ELK clusters while everyone in the team has other pressing\nresponsibilities is becoming a luxury we can’t afford. Another problem is that\nsome of the teams grow too fast and the ELK became inadequate and\noverwhelming to support enormous amount of logs.\nAccess\nHow we built ‘BARITO’ to enhance logging - 5 min read https://www.gojek.io/blog/how-we-built-barito-to-enhance-logging\n2 of 10 20/11/25, 10.27', '<2-hop>\n\nbigger, the ecosystem has also become more complex, resulting in both\ntechnical and managerial challenges for the developers.\nGoTo Financial head of Consumer Payment Infrastructure Engineering\n(InfEng) Giovanni Sakti and the company’s Developer Experience Technical\nlead Giri Kuncoro spoke about the challenges in the seventeenth episode of\nThat Digital Show , a weekly business podcast presented by Google Cloud on\ninnovation and growing value in a digital world.\nGiri explained that in the dynamic and fast-paced industry, developers were\nencouraged to innovate fast and build features quicker than competitors while\nalso accommodating the business needs. However, being fast was not the only\nrequirement, they also needed to build scalable and reliable systems.\nAddressing these technical issues, Giri shared that the team adopted modern\ncloud native technologies that could address the issues at scale, such as\nmicroservices, containers, Kubernetes and service mesh.\n“We did achieve the scale and reliability, but our infrastructure was becoming\nincreasingly more complex for developers who weren’t familiar with this\nlayer of technology, and ultimately slowed down our collective developer\nvelocity.”\nHe added that the cloud native adoption also required the developer teams to\nperform major infrastructure migration — multiple times. “Developers who\nwere supposed to deliver features to end customers had to deal with these\nunnecessary tasks. [...] When developers were not able to spend most of their\ntime writing codes for customers, it did not just affect their productivity but\nalso morale. Because they believed in tech for good, in making a lasting\nimpact on people.”\nGiovanni further stated that the risk of the company’s exponential growth\nwas that each of the tech leads would make decisions in silo. “With rapid\ngrowth, requirements pour in, so each tech lead would have to make decisions\non their own,” he said.\n\uf1e0\n\uf39e \uf099 \uf0e1\nGoPay.sh: A glimpse into Indonesia’s leading e-wallet GoPay’s Develo... https://www.thejakartapost.com/adv-longform/2022/06/16/gopay-sh-a-...\n2 of 5 20/11/25, 14.28', '<3-hop>\n\nH Ho ow w w we e b bu ui il lt t ‘ ‘B BA AR RI IT TO O’ ’ t to o e en nh ha an nc ce e l lo og gg gi in ng g\nBuilding a logging infrastructure commensurate to GO-JEK’s scale\n5 MIN READ \xa0| \xa0 DEC 08, 2018 Share: \nBB yy TT aa r r aa BB aa s s kk aa r r aa\nImagine coming home from a tiring day of work and your white-fenced house is\nsuddenly pink. You’d have questions. Many of them. Did I arrive at the correct\nhouse? Who did this? When did this happen? For what? And why pink? You’d\nlook around frantically for clues. Did any of your neighbours see anything? You’d\nHow we built ‘BARITO’ to enhance logging - 5 min read https://www.gojek.io/blog/how-we-built-barito-to-enhance-logging\n1 of 10 20/11/25, 10.27']","Logging is critical at GO-JEK because it provides visibility into how the system is doing and helps identify what went wrong when issues occur. Without it, finding problems is like finding a needle in a haystack, requiring checks across thousands of machines. The conventional ELK stack without sharding didn't scale for the quantity of logs at GO-JEK's scale. Teams had their own ELK clusters, but maintaining them became a luxury they couldn't afford, especially as teams grew and log volumes became overwhelming.",,,,multi_hop_abstract_query_synthesizer -"According to the RedSeer Industry Report released in December 2021, what percentage of the Indonesian population aged 15 and above were unbanked or underserved in 2020?","['<1-hop>\n\nGoPay, as part of Indonesia’s tech giant GoTo’s fintech provider GoTo\nFinancial, seeks to tackle this issue with technology.\nHowever, helping Indonesia’s population of 270 million to unlock their\nfinancial potential is not an easy task. At the same time, as GoPay has grown\nDespite holding the accolade of Southeast Asia’s largest economy,\nRedSeer Industry Report released in December 2021, showed that 55.5\npercent of the Indonesian population aged 15 and above were still\nunbanked or underserved in 2020.\nGoPay.sh: A glimpse into Indonesia’s\nleading\ne-wallet GoPay’s Developer Experience\n\uf1e0\n\uf39e \uf099 \uf0e1\nGoPay.sh: A glimpse into Indonesia’s leading e-wallet GoPay’s Develo... https://www.thejakartapost.com/adv-longform/2022/06/16/gopay-sh-a-...\n1 of 5 20/11/25, 14.28']","According to the RedSeer Industry Report released in December 2021, 55.5 percent of the Indonesian population aged 15 and above were still unbanked or underserved in 2020.",,,,multi_hop_specific_query_synthesizer +H Ho ow w w we e b bu ui il lt t ‘ ‘B BA AR RI IT TO O’ ’ t to o e en nh ha an nc ce e l lo og gg gi in ng g,"['H Ho ow w w we e b bu ui il lt t ‘ ‘B BA AR RI IT TO O’ ’ t to o e en nh ha an nc ce e l lo og gg gi in ng g\nBuilding a logging infrastructure commensurate to GO-JEK’s scale\n5 MIN READ \xa0| \xa0 DEC 08, 2018 Share: \nBB yy TT aa r r aa BB aa s s kk aa r r aa\nImagine coming home from a tiring day of work and your white-fenced house is\nsuddenly pink. You’d have questions. Many of them. Did I arrive at the correct\nhouse? Who did this? When did this happen? For what? And why pink? You’d\nlook around frantically for clues. Did any of your neighbours see anything? You’d\nHow we built ‘BARITO’ to enhance logging - 5 min read https://www.gojek.io/blog/how-we-built-barito-to-enhance-logging\n1 of 10 20/11/25, 10.27']","The logging infrastructure was designed to scale with GO-JEK’s operations, as described in the blog post on gojek.io.",Amina,MISSPELLED,MEDIUM,single_hop_specific_query_synthesizer +How d'd BARITO dev'pmt addr'ss ELK scal'g iss'z at GO-JEK?,"['<1-hop>\n\nH Ho ow w w we e b bu ui il lt t ‘ ‘B BA AR RI IT TO O’ ’ t to o e en nh ha an nc ce e l lo og gg gi in ng g\nBuilding a logging infrastructure commensurate to GO-JEK’s scale\n5 MIN READ \xa0| \xa0 DEC 08, 2018 Share: \nBB yy TT aa r r aa BB aa s s kk aa r r aa\nImagine coming home from a tiring day of work and your white-fenced house is\nsuddenly pink. You’d have questions. Many of them. Did I arrive at the correct\nhouse? Who did this? When did this happen? For what? And why pink? You’d\nlook around frantically for clues. Did any of your neighbours see anything? You’d\nHow we built ‘BARITO’ to enhance logging - 5 min read https://www.gojek.io/blog/how-we-built-barito-to-enhance-logging\n1 of 10 20/11/25, 10.27', '<2-hop>\n\nwant answers. So imagine how glad you’d be once you realized you’ve installed\nsecurity cameras in front of your house not long ago. All you have to do now is\ntake a look at the log and get all the answers.\nNow, what about complex applications and infrastructures systems? That’s\nthousands of white fences turning into every colour imaginable. How can you\nensure security cams record everything? At this point you’d agree that ll o o g g g g ii n n g g is\none of the most critical parts of any system. It gives us the visibility on how a\nsystem is doing and the ability to identify what went wrong when something\ngoes haywire.\nWhat about GO-JEK? Well, the absence of logging infrastructure brings us\nnothing but havoc. We’ve been through more than our fair and we critically need\nto improve in some areas:\nMean Time T o Respond (MTTR)\nAt GO-JEK, we need to improve our MTTR. We have millions of bookings each\nday. If our MTTR is high, can you imagine the number of drivers missing potential\norders? The number of people who may now be late for their appointments with\neach additional minute we take to solve an issue? Without proper logging, it’s\nakin to finding a needle in a haystack — because you have to check across\nthousands of machines in the faint hope that you might just stumble across the\nproblem.\nELK scale\nAt GO-JEK’s scale, the conventional ELK stack (without sharding) we used did\nnot scale for the quantity of logs. Each team has their own ELK cluster, but\nmaintaining ELK clusters while everyone in the team has other pressing\nresponsibilities is becoming a luxury we can’t afford. Another problem is that\nsome of the teams grow too fast and the ELK became inadequate and\noverwhelming to support enormous amount of logs.\nAccess\nHow we built ‘BARITO’ to enhance logging - 5 min read https://www.gojek.io/blog/how-we-built-barito-to-enhance-logging\n2 of 10 20/11/25, 10.27']","BARITO dev'pmt addr'ss'd ELK scal'g iss'z by provid'n a scal'ble logging sol'n for GO-JEK's infras'ure, reducin' MTTR an' managin' vast log vol'mes acr' ssyst'ms.",,,,multi_hop_abstract_query_synthesizer +"How did GoTo Financial address technical challenges in scaling their ecosystem while expanding financial inclusion in Indonesia, and what solutions did they adopt to improve developer productivity?","['<1-hop>\n\nGoPay, as part of Indonesia’s tech giant GoTo’s fintech provider GoTo\nFinancial, seeks to tackle this issue with technology.\nHowever, helping Indonesia’s population of 270 million to unlock their\nfinancial potential is not an easy task. At the same time, as GoPay has grown\nDespite holding the accolade of Southeast Asia’s largest economy,\nRedSeer Industry Report released in December 2021, showed that 55.5\npercent of the Indonesian population aged 15 and above were still\nunbanked or underserved in 2020.\nGoPay.sh: A glimpse into Indonesia’s\nleading\ne-wallet GoPay’s Developer Experience\n\uf1e0\n\uf39e \uf099 \uf0e1\nGoPay.sh: A glimpse into Indonesia’s leading e-wallet GoPay’s Develo... https://www.thejakartapost.com/adv-longform/2022/06/16/gopay-sh-a-...\n1 of 5 20/11/25, 14.28', '<2-hop>\n\nbigger, the ecosystem has also become more complex, resulting in both\ntechnical and managerial challenges for the developers.\nGoTo Financial head of Consumer Payment Infrastructure Engineering\n(InfEng) Giovanni Sakti and the company’s Developer Experience Technical\nlead Giri Kuncoro spoke about the challenges in the seventeenth episode of\nThat Digital Show , a weekly business podcast presented by Google Cloud on\ninnovation and growing value in a digital world.\nGiri explained that in the dynamic and fast-paced industry, developers were\nencouraged to innovate fast and build features quicker than competitors while\nalso accommodating the business needs. However, being fast was not the only\nrequirement, they also needed to build scalable and reliable systems.\nAddressing these technical issues, Giri shared that the team adopted modern\ncloud native technologies that could address the issues at scale, such as\nmicroservices, containers, Kubernetes and service mesh.\n“We did achieve the scale and reliability, but our infrastructure was becoming\nincreasingly more complex for developers who weren’t familiar with this\nlayer of technology, and ultimately slowed down our collective developer\nvelocity.”\nHe added that the cloud native adoption also required the developer teams to\nperform major infrastructure migration — multiple times. “Developers who\nwere supposed to deliver features to end customers had to deal with these\nunnecessary tasks. [...] When developers were not able to spend most of their\ntime writing codes for customers, it did not just affect their productivity but\nalso morale. Because they believed in tech for good, in making a lasting\nimpact on people.”\nGiovanni further stated that the risk of the company’s exponential growth\nwas that each of the tech leads would make decisions in silo. “With rapid\ngrowth, requirements pour in, so each tech lead would have to make decisions\non their own,” he said.\n\uf1e0\n\uf39e \uf099 \uf0e1\nGoPay.sh: A glimpse into Indonesia’s leading e-wallet GoPay’s Develo... https://www.thejakartapost.com/adv-longform/2022/06/16/gopay-sh-a-...\n2 of 5 20/11/25, 14.28']","GoTo Financial addressed technical challenges in scaling their ecosystem by adopting cloud-native technologies such as microservices, containers, Kubernetes, and service mesh to achieve scalability and reliability. However, this approach increased infrastructure complexity, slowing down developer velocity as teams struggled with unfamiliar technologies and repeated infrastructure migrations. To improve developer productivity, the company focused on aligning technical decisions with business needs while managing siloed decision-making across tech leads. These efforts aimed to support GoPay's mission of financial inclusion for Indonesia's unbanked population, despite the challenges of serving 270 million people with limited access to traditional banking services.",,,,multi_hop_specific_query_synthesizer diff --git a/evaluator/embeddings.py b/evaluator/embeddings.py index 300a282..5d47655 100644 --- a/evaluator/embeddings.py +++ b/evaluator/embeddings.py @@ -1,6 +1,22 @@ import requests from typing import List from langchain_core.embeddings import Embeddings +from langchain_ollama import OllamaEmbeddings +from ragas.embeddings import BaseRagasEmbeddings + +class OllamaRagasEmbeddings(OllamaEmbeddings): + # need to implement BaseRagasEmbeddings https://docs.ragas.io/en/stable/references/embeddings/ + + def __init__(self, model: str): + super().__init__(model=model) + + async def embed_text(self, text: str, **kwargs) -> List[float]: + return self.aembed_query(text) + + async def aembed_text(self, text: str, **kwargs) -> List[float]: + return self.aembed_query(text) + + class ModelGardenEmbeddings(Embeddings): api_url: str diff --git a/evaluator/generate-tests.ipynb b/evaluator/generate-tests.ipynb index 2889b6d..8bd7343 100644 --- a/evaluator/generate-tests.ipynb +++ b/evaluator/generate-tests.ipynb @@ -89,7 +89,7 @@ "source": [ "def search(query, collection, threshold=0.5, limit=3):\n", " query_vectors = embedding_fn.encode_queries([query])\n", - " res = client.search(\n", + " results = client.search(\n", " collection_name=collection, # target collection\n", " data=query_vectors, # query vectors\n", " limit=limit, # number of returned entities\n", @@ -100,8 +100,8 @@ " distanceThreshold = threshold\n", " \n", " return [\n", - " i\n", - " for i in res if i['distance'] >= distanceThreshold\n", + " result\n", + " for result in results if result['distance'] >= distanceThreshold\n", " ]" ] }, @@ -129,15 +129,18 @@ } ], "source": [ - "from langchain_community.document_loaders import PyPDFLoader\n", + "from langchain_community.document_loaders import FileSystemBlobLoader\n", + "from langchain_community.document_loaders.generic import GenericLoader\n", + "from langchain_community.document_loaders.parsers import PyPDFParser\n", "\n", - "docs = []\n", - "\n", - "for root, _, files in os.walk(\"../datasets\"):\n", - " for file in files:\n", - " if file.endswith('.pdf'):\n", - " loader = PyPDFLoader(root + '/' + file)\n", - " docs.extend(loader.load())" + "loader = GenericLoader(\n", + " blob_loader=FileSystemBlobLoader(\n", + " path=\"../datasets\",\n", + " glob=\"**/*.pdf\",\n", + " ),\n", + " blob_parser=PyPDFParser(),\n", + ")\n", + "docs = loader.load()" ] }, { @@ -179,8 +182,7 @@ "\n", "is_debug = False\n", "\n", - "for i in range(len(chunks)):\n", - " chunk = chunks[i]\n", + "for i, chunk in enumerate(chunks):\n", " vector = embedding_fn.encode_documents([chunk.page_content])\n", " d = {\n", " \"id\": i,\n", @@ -229,51 +231,50 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "id": "50190723-5cb6-47d2-8aa7-884183987742", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "'ollama'\n" + ] + } + ], "source": [ "from llm import ModelGardenLLM\n", - "from embeddings import ModelGardenEmbeddings\n", + "from embeddings import ModelGardenEmbeddings, OllamaRagasEmbeddings\n", + "from langchain_ollama import OllamaLLM, OllamaEmbeddings\n", "\n", - "url = os.getenv('MODEL_GARDEN_URL')\n", + "llm_type = os.getenv('LLM_TYPE')\n", "model = os.getenv('MODEL_GARDEN_MODEL')\n", - "embed_url = os.getenv('EMBEDDING_URL')\n", "embedding = os.getenv('EMBEDDING_MODEL')\n", "\n", - "llm = ModelGardenLLM(api_url=url, model=model)\n", - "embeds = ModelGardenEmbeddings(api_url=embed_url, model=embedding)" + "if llm_type == \"model_garden\":\n", + " url = os.getenv('MODEL_GARDEN_URL')\n", + " embed_url = os.getenv('EMBEDDING_URL')\n", + " llm = ModelGardenLLM(api_url=url, model=model)\n", + " embeds = ModelGardenEmbeddings(api_url=embed_url, model=embedding)\n", + "elif llm_type == \"ollama\":\n", + " llm = OllamaLLM(model=model, temperature=0.4)\n", + " embeds = OllamaRagasEmbeddings(model=embedding)\n", + "else:\n", + " raise ValueError(f\"Unsupported LLM type: {llm_type}\")\n", + "pp(llm_type)" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "7d2018f3-93fc-4ccf-a75e-e27541f4659c", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" - ] - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8da6ab7cb7bb4f548ae2560637cb8ce9", + "model_id": "15b56c28318a43beb0546f6809510e3f", "version_major": 2, "version_minor": 0 }, @@ -287,7 +288,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2ac2d0c48c6943dc8d8933c75823d0ca", + "model_id": "1867eacde72c463fb7bc9a56016c5321", "version_major": 2, "version_minor": 0 }, @@ -301,7 +302,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "78ec54d621f14813bf3eb372990992b2", + "model_id": "acff15a1fc824570b425b14188c6abea", "version_major": 2, "version_minor": 0 }, @@ -312,10 +313,18 @@ "metadata": {}, "output_type": "display_data" }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/williamdembo/projects/github.com/walbertus/rag-pipeline-poc/evaluator/.venv/lib/python3.13/site-packages/ragas/testset/transforms/base.py:202: UserWarning: Using sync embedding model OllamaRagasEmbeddings in async context. This may impact performance. Consider using an async-compatible embedding model for better performance.\n", + " property_name, property_value = await self.extract(node)\n" + ] + }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6507670f5d4a47759f043353fd710e33", + "model_id": "9272678788954b44b4cd436060211e5b", "version_major": 2, "version_minor": 0 }, @@ -329,7 +338,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4fff493628d14b6aa078e75f07b264a9", + "model_id": "037527597d3243ffb41f8ee5d9242989", "version_major": 2, "version_minor": 0 }, @@ -343,7 +352,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "92f6e414419e4d49a4188817f36232b5", + "model_id": "2feb386ccca94525a4c7e2e6048bc7aa", "version_major": 2, "version_minor": 0 }, @@ -357,7 +366,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "020877be2e6c4c6990a75218d824559d", + "model_id": "33fda6474e4f45fcbdbbcbee649a00b1", "version_major": 2, "version_minor": 0 }, @@ -371,7 +380,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ad85c0a2770941d7ab6586d9aeaaf760", + "model_id": "3a180672957e488892814a8020a77682", "version_major": 2, "version_minor": 0 }, @@ -385,7 +394,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b6ef2f5411444ad8aa3b05327849e42b", + "model_id": "72ba7660601d4201a9a37ba71c83161b", "version_major": 2, "version_minor": 0 }, @@ -399,7 +408,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4ed0f97867084fbc8349cddca547cf1f", + "model_id": "08e8327d65284c578382d79fe447e3a1", "version_major": 2, "version_minor": 0 }, @@ -458,19 +467,19 @@ " \n", " \n", " 0\n", - " According to the provided text, what was the p...\n", + " H Ho ow w w we e b bu ui il lt t ‘ ‘B BA AR...\n", " [H Ho ow w w we e b bu ui il lt t ‘ ‘B BA A...\n", - " ‘BARITO’ was built to enhance logging.\n", - " DevOps Engineer - GO-JEK\n", - " PERFECT_GRAMMAR\n", + " The logging infrastructure was designed to sca...\n", + " Amina\n", + " MISSPELLED\n", " MEDIUM\n", " single_hop_specific_query_synthesizer\n", " \n", " \n", " 1\n", - " logging infrastructure at GO-JEK, like, why is...\n", - " [<1-hop>\\n\\nwant answers. So imagine how glad ...\n", - " Logging is critical at GO-JEK because it provi...\n", + " How d'd BARITO dev'pmt addr'ss ELK scal'g iss'...\n", + " [<1-hop>\\n\\nH Ho ow w w we e b bu ui il lt t...\n", + " BARITO dev'pmt addr'ss'd ELK scal'g iss'z by p...\n", " NaN\n", " NaN\n", " NaN\n", @@ -478,9 +487,9 @@ " \n", " \n", " 2\n", - " According to the RedSeer Industry Report relea...\n", + " How did GoTo Financial address technical chall...\n", " [<1-hop>\\n\\nGoPay, as part of Indonesia’s tech...\n", - " According to the RedSeer Industry Report relea...\n", + " GoTo Financial addressed technical challenges ...\n", " NaN\n", " NaN\n", " NaN\n", @@ -492,29 +501,24 @@ ], "text/plain": [ " user_input \\\n", - "0 According to the provided text, what was the p... \n", - "1 logging infrastructure at GO-JEK, like, why is... \n", - "2 According to the RedSeer Industry Report relea... \n", + "0 H Ho ow w w we e b bu ui il lt t ‘ ‘B BA AR... \n", + "1 How d'd BARITO dev'pmt addr'ss ELK scal'g iss'... \n", + "2 How did GoTo Financial address technical chall... \n", "\n", " reference_contexts \\\n", "0 [H Ho ow w w we e b bu ui il lt t ‘ ‘B BA A... \n", - "1 [<1-hop>\\n\\nwant answers. So imagine how glad ... \n", + "1 [<1-hop>\\n\\nH Ho ow w w we e b bu ui il lt t... \n", "2 [<1-hop>\\n\\nGoPay, as part of Indonesia’s tech... \n", "\n", - " reference \\\n", - "0 ‘BARITO’ was built to enhance logging. \n", - "1 Logging is critical at GO-JEK because it provi... \n", - "2 According to the RedSeer Industry Report relea... \n", - "\n", - " persona_name query_style query_length \\\n", - "0 DevOps Engineer - GO-JEK PERFECT_GRAMMAR MEDIUM \n", - "1 NaN NaN NaN \n", - "2 NaN NaN NaN \n", + " reference persona_name query_style \\\n", + "0 The logging infrastructure was designed to sca... Amina MISSPELLED \n", + "1 BARITO dev'pmt addr'ss'd ELK scal'g iss'z by p... NaN NaN \n", + "2 GoTo Financial addressed technical challenges ... NaN NaN \n", "\n", - " synthesizer_name \n", - "0 single_hop_specific_query_synthesizer \n", - "1 multi_hop_abstract_query_synthesizer \n", - "2 multi_hop_specific_query_synthesizer " + " query_length synthesizer_name \n", + "0 MEDIUM single_hop_specific_query_synthesizer \n", + "1 NaN multi_hop_abstract_query_synthesizer \n", + "2 NaN multi_hop_specific_query_synthesizer " ] }, "execution_count": 14, @@ -525,11 +529,19 @@ "source": [ "dataset.to_pandas()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3bd13981-de96-4d81-b7d1-dedc57a05207", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "evaluator", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -543,7 +555,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.7" + "version": "3.13.9" } }, "nbformat": 4, diff --git a/evaluator/pyproject.toml b/evaluator/pyproject.toml index 5072850..cb516d0 100644 --- a/evaluator/pyproject.toml +++ b/evaluator/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "langchain>=1.0.7", "langchain-community>=0.4.1", "langchain-core>=1.0.4", + "langchain-ollama>=1.0.0", "langchain-text-splitters>=1.0.0", "milvus-lite>=2.5.1", "pymilvus>=2.6.3", diff --git a/evaluator/uv.lock b/evaluator/uv.lock index 14a51b5..0868a61 100644 --- a/evaluator/uv.lock +++ b/evaluator/uv.lock @@ -529,6 +529,7 @@ dependencies = [ { name = "langchain" }, { name = "langchain-community" }, { name = "langchain-core" }, + { name = "langchain-ollama" }, { name = "langchain-text-splitters" }, { name = "milvus-lite" }, { name = "pymilvus" }, @@ -551,6 +552,7 @@ requires-dist = [ { name = "langchain", specifier = ">=1.0.7" }, { name = "langchain-community", specifier = ">=0.4.1" }, { name = "langchain-core", specifier = ">=1.0.4" }, + { name = "langchain-ollama", specifier = ">=1.0.0" }, { name = "langchain-text-splitters", specifier = ">=1.0.0" }, { name = "milvus-lite", specifier = ">=2.5.1" }, { name = "pymilvus", specifier = ">=2.6.3" }, @@ -1433,6 +1435,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/d0/3e7b87b929e95310a219206937f614796d266bfc5e4350c32c5d6502c183/langchain_core-1.0.7-py3-none-any.whl", hash = "sha256:76af258b0e95a7915b8e301706a45ded50c75b80ff35329394d4df964416e32a", size = 472951, upload-time = "2025-11-19T21:23:26.579Z" }, ] +[[package]] +name = "langchain-ollama" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "ollama" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/45/d996b4c0e3e7155ff7a110bf24dcb5043fc1405559a2089c11fe97511cc2/langchain_ollama-1.0.0.tar.gz", hash = "sha256:2ea9ad1b0f0ab319d600b9193d1124a8925523a3b943d75a967718e24ec09a8a", size = 151042, upload-time = "2025-10-17T15:41:50.277Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/08/7be292aee722692b13a93316247b57eefb83d4309f5fdfe636cc47786efe/langchain_ollama-1.0.0-py3-none-any.whl", hash = "sha256:5828523fcbd137847490841110a6aedf96b68534e7fe2735715ecf3e835b2391", size = 29006, upload-time = "2025-10-17T15:41:49.497Z" }, +] + [[package]] name = "langchain-openai" version = "1.0.3" @@ -1940,6 +1955,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2d/fd/4b5eb0b3e888d86aee4d198c23acec7d214baaf17ea93c1adec94c9518b9/numpy-2.3.5-cp314-cp314t-win_arm64.whl", hash = "sha256:6203fdf9f3dc5bdaed7319ad8698e685c7a3be10819f41d32a0723e611733b42", size = 10545459, upload-time = "2025-11-16T22:52:20.55Z" }, ] +[[package]] +name = "ollama" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/5a/652dac4b7affc2b37b95386f8ae78f22808af09d720689e3d7a86b6ed98e/ollama-0.6.1.tar.gz", hash = "sha256:478c67546836430034b415ed64fa890fd3d1ff91781a9d548b3325274e69d7c6", size = 51620, upload-time = "2025-11-13T23:02:17.416Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/4f/4a617ee93d8208d2bcf26b2d8b9402ceaed03e3853c754940e2290fed063/ollama-0.6.1-py3-none-any.whl", hash = "sha256:fc4c984b345735c5486faeee67d8a265214a31cbb828167782dc642ce0a2bf8c", size = 14354, upload-time = "2025-11-13T23:02:16.292Z" }, +] + [[package]] name = "onnxruntime" version = "1.23.2"