From 9a8026c2e0372e67527ec6b864d731bdf98a243f Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:25:47 -0600
Subject: [PATCH 01/60] feat(tests): add requirements-dev.txt

---
 requirements-dev.txt | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 requirements-dev.txt

diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000000..e9fdfc8867
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,7 @@
+black==24.2.0
+pre-commit==3.7.0
+pytest==6.2.5
+pytest-mock==3.12.0
+ruff==0.3.0
+torch==2.2.1
+transformers==4.38.2

From 35dcaed2e1b01a6f46fd322982a50800fb339c25 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:27:45 -0600
Subject: [PATCH 02/60] feat(tests): BaseIntegrationTestWithCache

---
 tests_integration/base.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 tests_integration/base.py

diff --git a/tests_integration/base.py b/tests_integration/base.py
new file mode 100644
index 0000000000..ef079d9e38
--- /dev/null
+++ b/tests_integration/base.py
@@ -0,0 +1,17 @@
+import os
+from pathlib import Path
+
+import pytest
+
+
+class BaseIntegrationTestWithCache:
+    @pytest.fixture(autouse=True)
+    def setup(self) -> None:
+        # Base directory for all DSPy modules
+        library_dir = Path(__file__).resolve().parent
+        base_dir = library_dir.parent
+        cache_dir = str(base_dir / "cache")
+        os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
+
+        if cache_dir and not Path(cache_dir).exists():
+            Path(cache_dir).mkdir(parents=True)

From a1c84916ca5910a4bb845a53033e5a970669dcfe Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:35:20 -0600
Subject: [PATCH 03/60] feat(tests): ignore assert in ruff

---
 pyproject.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 7557fdce05..8bf893bdfc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -284,6 +284,8 @@ ignore = [
     "E731",
     # Sometimes we need List and Tuple
     "UP006",
+    # Ignore assert
+    "S101",
 ]
 
 # Allow fix for all enabled rules (when `--fix`) is provided.

From 6f1c1209dac538f71cb36a1fd06e501fb8e04e94 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:35:37 -0600
Subject: [PATCH 04/60] feat(tests): TestIntroIntegration

---
 tests_integration/test_intro.py | 299 ++++++++++++++++++++++++++++++++
 1 file changed, 299 insertions(+)
 create mode 100644 tests_integration/test_intro.py

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
new file mode 100644
index 0000000000..39fb1ed51d
--- /dev/null
+++ b/tests_integration/test_intro.py
@@ -0,0 +1,299 @@
+from typing import Any
+
+from tests_integration.base import BaseIntegrationTestWithCache
+
+
+class TestIntroIntegration(BaseIntegrationTestWithCache):
+    def test_dspy_workflow(self) -> None:
+        dspy = self.setup_dspy()
+
+        dev_example, dev_set, training_set = self.assert_dataset_loading()
+
+        self.assert_basic_qa(dev_example, dspy)
+
+        self.assert_retrieval(dev_example, dspy)
+
+        self.assert_compilation(dev_set, dspy, training_set)
+
+    def assert_compilation(self, devset, dspy, trainset) -> None:
+        class GenerateAnswer(dspy.Signature):
+            """Answer questions with short factoid answers."""
+
+            context = dspy.InputField(desc="may contain relevant facts")
+            question = dspy.InputField()
+            answer = dspy.OutputField(desc="often between 1 and 5 words")
+
+        class RAG(dspy.Module):
+            def __init__(self, num_passages=3):
+                super().__init__()
+
+                self.retrieve = dspy.Retrieve(k=num_passages)
+                self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
+
+            def forward(self, question):
+                context = self.retrieve(question).passages
+                prediction = self.generate_answer(context=context, question=question)
+                return dspy.Prediction(context=context, answer=prediction.answer)
+
+        from dspy.teleprompt import BootstrapFewShot
+
+        # Validation logic: check that the predicted answer is correct.
+        # Also check that the retrieved context actually contains that answer.
+        def validate_context_and_answer(example, pred, trace=None):  # noqa
+            answer_em = dspy.evaluate.answer_exact_match(example, pred)
+            answer_pm = dspy.evaluate.answer_passage_match(example, pred)
+            return answer_em and answer_pm
+
+        # Set up a basic teleprompter, which will compile our RAG program.
+        teleprompter = BootstrapFewShot(metric=validate_context_and_answer)
+        # Compile the RAG model
+        compiled_rag = teleprompter.compile(RAG(), trainset=trainset)
+
+        # Test the compiled RAG model with a question
+        my_question = "What castle did David Gregory inherit?"
+        pred = compiled_rag(my_question)
+
+        # Assertions to verify the compiled RAG model
+        assert f"Question: {my_question}" == "Question: What castle did David Gregory inherit?"
+        assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: Kinnairdy Castle"
+        assert f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}" == (
+            "Retrieved Contexts (truncated): ['David Gregory (physician) | David Gregory (20 December 1625 – 1720) "
+            "was a Scottish physician and inventor."
+            "His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinn...', "
+            "'Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio "
+            'Tracanioto" or "Tracamoto" ) was a "protospatharius" and the long-reigning catepan of Italy from 998 '
+            "t...', "
+            "'David Gregory (mathematician) | David Gregory (originally spelt Gregorie) FRS (? 1659 – 10 October "
+            "1708) was a Scottish mathematician and astronomer."
+            "He was professor of mathematics at the University ...']"
+        )
+
+        # Verify compiled model's parameters
+        for name, parameter in compiled_rag.named_predictors():
+            assert name is not None
+            assert parameter.demos[0] is not None
+
+        from dspy.evaluate.evaluate import Evaluate
+
+        # Set up the evaluation function
+        evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)
+        # Evaluate the compiled RAG program with the exact match metric
+        metric = dspy.evaluate.answer_exact_match
+        evaluate_on_hotpotqa(compiled_rag, metric=metric)
+
+        def gold_passages_retrieved(example, pred, trace=None):  # noqa
+            gold_titles = set(map(dspy.evaluate.normalize_text, example["gold_titles"]))
+            found_titles = set(map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context]))
+            return gold_titles.issubset(found_titles)
+
+        compiled_rag_retrieval_score = evaluate_on_hotpotqa(compiled_rag, metric=gold_passages_retrieved)
+
+        class GenerateSearchQuery(dspy.Signature):
+            """Write a simple search query that will help answer a complex question."""
+
+            context = dspy.InputField(desc="may contain relevant facts")
+            question = dspy.InputField()
+            query = dspy.OutputField()
+
+        from dsp.utils import deduplicate
+
+        class SimplifiedBaleen(dspy.Module):
+            def __init__(self, passages_per_hop=3, max_hops=2):
+                super().__init__()
+
+                self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]
+                self.retrieve = dspy.Retrieve(k=passages_per_hop)
+                self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
+                self.max_hops = max_hops
+
+            def forward(self, question):
+                context = []
+
+                for hop in range(self.max_hops):
+                    query = self.generate_query[hop](context=context, question=question).query
+                    passages = self.retrieve(query).passages
+                    context = deduplicate(context + passages)
+
+                pred = self.generate_answer(context=context, question=question)
+                return dspy.Prediction(context=context, answer=pred.answer)
+
+        # Test the SimplifiedBaleen model with a question
+        my_question = "How many storeys are in the castle that David Gregory inherited?"
+        uncompiled_baleen = SimplifiedBaleen()
+        pred = uncompiled_baleen(my_question)
+
+        # Assertions to verify the SimplifiedBaleen model
+        assert (
+            f"Question: {my_question}" == "Question: How many storeys are in the castle that David Gregory inherited?"
+        )
+        assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: five"
+        assert f"Retrieved Contexts (truncated): {[c[:20] + '...' for c in pred.context]}" == (
+            "Retrieved Contexts (truncated): ['David Gregory (physi...', 'The Boleyn Inheritan...', 'Gregory of Gaeta "
+            "| G...',"
+            "'Kinnairdy Castle | K...', 'Kinnaird Head | Kinn...', 'Kinnaird Castle, Bre...']"
+        )
+
+        def validate_context_and_answer_and_hops(example, pred, trace=None):
+            if not dspy.evaluate.answer_exact_match(example, pred):
+                return False
+            if not dspy.evaluate.answer_passage_match(example, pred):
+                return False
+
+            hops = [example.question] + [outputs.query for *_, outputs in trace if "query" in outputs]
+
+            if max([len(h) for h in hops]) > 100:
+                return False
+            if any(
+                dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))
+            ):
+                return False
+
+            return True
+
+        teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)
+        compiled_baleen = teleprompter.compile(
+            SimplifiedBaleen(),
+            teacher=SimplifiedBaleen(passages_per_hop=2),
+            trainset=trainset,
+        )
+        uncompiled_baleen_retrieval_score = evaluate_on_hotpotqa(uncompiled_baleen, metric=gold_passages_retrieved)
+        compiled_baleen_retrieval_score = evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved)
+
+        # Assertions for the retrieval scores
+        assert f"## Retrieval Score for RAG: {compiled_rag_retrieval_score}" == "## Retrieval Score for RAG: 26.0"
+        assert (
+            f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}"
+            == "## Retrieval Score for uncompiled Baleen: 36.0"
+        )
+        assert (
+            f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}"
+            == "## Retrieval Score for compiled Baleen: 60.0"
+        )
+        assert compiled_baleen("How many storeys are in the castle that David Gregory inherited?") is not None
+
+    def assert_retrieval(self, dev_example, dspy) -> None:
+        retrieve = dspy.Retrieve(k=3)
+        top_k_passages = retrieve(dev_example.question).passages
+
+        # Assertions to verify the retrieval functionality
+        assert retrieve.k == 3
+        assert (
+            dev_example.question
+            == "What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+        )
+        assert (
+            top_k_passages[0]
+            == "Restaurant: Impossible | Restaurant: Impossible is an American reality television series, featuring "
+            "chef and restaurateur Robert Irvine, that aired on Food Network from 2011 to 2016."
+        )
+        assert (
+            top_k_passages[1]
+            == "Jean Joho | Jean Joho is a French-American chef and restaurateur. He is chef/proprietor of Everest in "
+            "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
+            "Restaurant in Las Vegas, and Brasserie JO in Boston."
+        )
+        assert top_k_passages[2] == (
+            "List of Restaurant: Impossible episodes | This is the list of the episodes for the American cooking and "
+            'reality television series "Restaurant Impossible", '
+            "produced by Food Network. The premise of the series is that within two days and on a budget of $10,000, "
+            "celebrity chef Robert Irvine renovates a failing American restaurant with the goal of helping to restore "
+            "it to profitability and prominence."
+            "Irvine is assisted by a designer (usually Taniya Nayak, Cheryl Torrenueva, or Lynn Keagan, but sometimes "
+            "Vanessa De Leon, Krista Watterworth, Yvette Irene, or Nicole Faccuito), along with general contractor "
+            "Tom Bury, who sometimes does double duty as both general contractor and designer."
+            "After assessing the problems with the restaurant, Robert Irvine typically creates a plan for the new "
+            "decor, oversees the cleaning of the restaurant, reduces the size of the menu and improves the food, "
+            "develops a promotional activity, educates the restaurant's owners, or trains the staff, as needed by "
+            "each restaurant."
+        )
+
+        retrieved_value = retrieve("When was the first FIFA World Cup held?").passages[0]
+        assert retrieved_value == (
+            "History of the FIFA World Cup | The FIFA World Cup was first held in 1930, when FIFA president Jules "
+            "Rimet decided to stage an international football tournament."
+            "The inaugural edition, held in 1930, was contested as a final tournament of only thirteen teams invited "
+            "by the organization."
+            "Since then, the World Cup has experienced successive expansions and format remodeling to its current "
+            "32-team final tournament preceded by a two-year qualifying process, involving over 200 teams from around "
+            "the world."
+        )
+
+    def assert_basic_qa(self, dev_example, dspy) -> None:
+        class BasicQA(dspy.Signature):
+            """Answer questions with short factoid answers."""
+
+            question = dspy.InputField()
+            answer = dspy.OutputField(desc="often between 1 and 5 words")
+
+        # Define the predictor
+        generate_answer = dspy.Predict(BasicQA)
+        # Call the predictor on a particular input
+        pred = generate_answer(question=dev_example.question)
+
+        # Assertions to verify the basic QA functionality
+        assert (
+            f"Question: {dev_example.question}"
+            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+        )
+        assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: American"
+
+        # Define the predictor with chain of thought
+        generate_answer_with_chain_of_thought = dspy.ChainOfThought(BasicQA)
+        # Call the predictor on the same input
+        pred = generate_answer_with_chain_of_thought(question=dev_example.question)
+
+        # Assertions to verify the chain of thought functionality
+        assert (
+            f"Question: {dev_example.question}"
+            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+        )
+        assert (
+            f"Thought: {pred.rationale.split('.', 1)[1].strip()}"
+            == "Thought: We know that the chef and restaurateur featured in Restaurant: Impossible is Robert Irvine."
+        )
+        assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: British"
+
+    def assert_dataset_loading(self) -> None:
+        from dspy.datasets import HotPotQA
+
+        # Load the dataset
+        dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)
+
+        # Prepare the datasets for training and development
+        trainset = [x.with_inputs("question") for x in dataset.train]
+        devset = [x.with_inputs("question") for x in dataset.dev]
+        train_example = trainset[0]
+
+        # Assertions to verify the dataset loading
+        assert (
+            f"Question: {train_example.question}"
+            == "Question: At My Window was released by which American singer-songwriter?"
+        )
+        assert f"Answer: {train_example.answer}" == "Answer: John Townes Van Zandt"
+        dev_example = devset[18]
+        assert (
+            f"Question: {dev_example.question}"
+            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+        )
+        assert f"Answer: {dev_example.answer}" == "Answer: English"
+        assert "Restaurant: Impossible" in list(dev_example.gold_titles)
+        assert "Robert Irvine" in list(dev_example.gold_titles)
+        assert (
+            f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys "
+            f"{train_example.labels().keys()}"
+            == "For this dataset, training examples have input keys ['question'] and label keys ['answer']"
+        )
+        assert (
+            f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys "
+            f"{dev_example.labels().keys()}"
+            == "For this dataset, dev examples have input keys ['question'] and label keys ['answer', 'gold_titles']"
+        )
+        return dev_example, devset, trainset
+
+    def setup_dspy(self) -> Any:
+        import dspy
+
+        turbo = dspy.OpenAI(model="gpt-3.5-turbo")
+        colbertv2_wiki17_abstracts = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")
+        dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)
+        return dspy

From b77379676dc561e30353ef85469a29f6f0330443 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:35:54 -0600
Subject: [PATCH 05/60] feat(tests): TestIntroIntegration

---
 tests_integration/test_intro.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index 39fb1ed51d..bea8b3ace0 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -253,7 +253,7 @@ class BasicQA(dspy.Signature):
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: British"
 
-    def assert_dataset_loading(self) -> None:
+    def assert_dataset_loading(self) -> tuple:
         from dspy.datasets import HotPotQA
 
         # Load the dataset

From 32b5470579f2487846f85f71c256ae4079332d24 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:45:19 -0600
Subject: [PATCH 06/60] feat(tests): TestIntroIntegration

---
 tests_integration/test_intro.py | 53 ++++++++++-----------------------
 1 file changed, 15 insertions(+), 38 deletions(-)

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index bea8b3ace0..ec0c8c7511 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -56,16 +56,9 @@ def validate_context_and_answer(example, pred, trace=None):  # noqa
         # Assertions to verify the compiled RAG model
         assert f"Question: {my_question}" == "Question: What castle did David Gregory inherit?"
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: Kinnairdy Castle"
-        assert f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}" == (
-            "Retrieved Contexts (truncated): ['David Gregory (physician) | David Gregory (20 December 1625 – 1720) "
-            "was a Scottish physician and inventor."
-            "His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinn...', "
-            "'Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio "
-            'Tracanioto" or "Tracamoto" ) was a "protospatharius" and the long-reigning catepan of Italy from 998 '
-            "t...', "
-            "'David Gregory (mathematician) | David Gregory (originally spelt Gregorie) FRS (? 1659 – 10 October "
-            "1708) was a Scottish mathematician and astronomer."
-            "He was professor of mathematics at the University ...']"
+        assert (
+            f"Retrieved Contexts (truncated): {[c[:10] + '...' for c in pred.context]}"
+            == "Retrieved Contexts (truncated): ['David Greg...', 'Gregory Ta...', 'David Greg...']"
         )
 
         # Verify compiled model's parameters
@@ -127,11 +120,16 @@ def forward(self, question):
             f"Question: {my_question}" == "Question: How many storeys are in the castle that David Gregory inherited?"
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: five"
-        assert f"Retrieved Contexts (truncated): {[c[:20] + '...' for c in pred.context]}" == (
-            "Retrieved Contexts (truncated): ['David Gregory (physi...', 'The Boleyn Inheritan...', 'Gregory of Gaeta "
-            "| G...',"
-            "'Kinnairdy Castle | K...', 'Kinnaird Head | Kinn...', 'Kinnaird Castle, Bre...']"
-        )
+
+        # Retrieved Contexts (truncated)
+        assert [c[:10] for c in pred.context] == [
+            "David Greg",
+            "The Boleyn",
+            "Gregory of",
+            "Kinnairdy ",
+            "Kinnaird H",
+            "Kinnaird C",
+        ]
 
         def validate_context_and_answer_and_hops(example, pred, trace=None):
             if not dspy.evaluate.answer_exact_match(example, pred):
@@ -192,31 +190,10 @@ def assert_retrieval(self, dev_example, dspy) -> None:
             "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
             "Restaurant in Las Vegas, and Brasserie JO in Boston."
         )
-        assert top_k_passages[2] == (
-            "List of Restaurant: Impossible episodes | This is the list of the episodes for the American cooking and "
-            'reality television series "Restaurant Impossible", '
-            "produced by Food Network. The premise of the series is that within two days and on a budget of $10,000, "
-            "celebrity chef Robert Irvine renovates a failing American restaurant with the goal of helping to restore "
-            "it to profitability and prominence."
-            "Irvine is assisted by a designer (usually Taniya Nayak, Cheryl Torrenueva, or Lynn Keagan, but sometimes "
-            "Vanessa De Leon, Krista Watterworth, Yvette Irene, or Nicole Faccuito), along with general contractor "
-            "Tom Bury, who sometimes does double duty as both general contractor and designer."
-            "After assessing the problems with the restaurant, Robert Irvine typically creates a plan for the new "
-            "decor, oversees the cleaning of the restaurant, reduces the size of the menu and improves the food, "
-            "develops a promotional activity, educates the restaurant's owners, or trains the staff, as needed by "
-            "each restaurant."
-        )
+        assert top_k_passages[2][:30] == "List of Restaurant: Impossible"
 
         retrieved_value = retrieve("When was the first FIFA World Cup held?").passages[0]
-        assert retrieved_value == (
-            "History of the FIFA World Cup | The FIFA World Cup was first held in 1930, when FIFA president Jules "
-            "Rimet decided to stage an international football tournament."
-            "The inaugural edition, held in 1930, was contested as a final tournament of only thirteen teams invited "
-            "by the organization."
-            "Since then, the World Cup has experienced successive expansions and format remodeling to its current "
-            "32-team final tournament preceded by a two-year qualifying process, involving over 200 teams from around "
-            "the world."
-        )
+        assert retrieved_value[:30] == "History of the FIFA World Cup "
 
     def assert_basic_qa(self, dev_example, dspy) -> None:
         class BasicQA(dspy.Signature):

From 2e10d67ff17ff0a6127cd8b3a5ac939b1f6aab05 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:51:21 -0600
Subject: [PATCH 07/60] feat(tests): add integration test to CI.

---
 .github/workflows/run_tests.yml | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 07062257a3..e8ae968308 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -65,6 +65,37 @@ jobs:
       - name: Run tests with pytest
         run: poetry run pytest tests/
 
+  integration_test:
+    name: Run Integration Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Load cached Poetry installation
+        id: cached-poetry
+        uses: actions/cache@v3
+        with:
+          path: ~/.local
+          key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
+      - name: Install Poetry
+        if: steps.cached-poetry.outputs.cache-hit != 'true'
+        uses: snok/install-poetry@v1
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "poetry"
+      - name: Install dependencies
+        run: poetry install --no-interaction
+      - name: Run lint with tests
+        uses: chartboost/ruff-action@v1
+        with:
+          args: --fix-only
+      - name: Run tests with pytest
+        run: poetry run pytest tests_integration/
+
   build_poetry:
     name: Build Poetry
     runs-on: ubuntu-latest

From 6df93fce35063c4ecf2a7fc397d78f2f15814152 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:57:55 -0600
Subject: [PATCH 08/60] feat(tests): add integration test to CI.

---
 .github/workflows/run_tests.yml | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index e8ae968308..dd4a65eb1d 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -69,32 +69,20 @@ jobs:
     name: Run Integration Tests
     runs-on: ubuntu-latest
     strategy:
-      matrix:
-        python-version: ["3.9"]
+    matrix:
+      python-version: ["3.9"]
     steps:
       - uses: actions/checkout@v4
-      - name: Load cached Poetry installation
-        id: cached-poetry
-        uses: actions/cache@v3
-        with:
-          path: ~/.local
-          key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
-      - name: Install Poetry
-        if: steps.cached-poetry.outputs.cache-hit != 'true'
-        uses: snok/install-poetry@v1
       - name: Set up python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "poetry"
       - name: Install dependencies
-        run: poetry install --no-interaction
-      - name: Run lint with tests
-        uses: chartboost/ruff-action@v1
-        with:
-          args: --fix-only
+        run: pip install -r requirements.txt
+      - name: Install dev dependencies
+        run: pip install -r requirements-dev.txt
       - name: Run tests with pytest
-        run: poetry run pytest tests_integration/
+        run: pytest tests_integration/
 
   build_poetry:
     name: Build Poetry

From 195cd03292ffb0d59c134475d410a1116e4d8237 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 18:58:47 -0600
Subject: [PATCH 09/60] feat(tests): add integration test to CI.

---
 .github/workflows/run_tests.yml | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index dd4a65eb1d..07062257a3 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -65,25 +65,6 @@ jobs:
       - name: Run tests with pytest
         run: poetry run pytest tests/
 
-  integration_test:
-    name: Run Integration Tests
-    runs-on: ubuntu-latest
-    strategy:
-    matrix:
-      python-version: ["3.9"]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: pip install -r requirements.txt
-      - name: Install dev dependencies
-        run: pip install -r requirements-dev.txt
-      - name: Run tests with pytest
-        run: pytest tests_integration/
-
   build_poetry:
     name: Build Poetry
     runs-on: ubuntu-latest

From 965fbf35109cd61ce791f8e97badeccab397791f Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 19:01:54 -0600
Subject: [PATCH 10/60] feat(tests): add integration test to CI.

---
 .github/workflows/run_tests.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 07062257a3..974dd9c68a 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -120,3 +120,22 @@ jobs:
           cache: "poetry"
       - name: Run setup.py build
         run: python setup.py build
+
+  integration_test:
+    name: Run Integration Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Install dev dependencies
+        run: pip install -r requirements-dev.txt
+      - name: Run tests with pytest
+        run: pytest tests_integration/

From ff075fed549975954c3c91a3c15504f7cf4962f1 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 19:06:28 -0600
Subject: [PATCH 11/60] feat(tests): add integration test to CI.

---
 .github/workflows/run_tests.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 974dd9c68a..b94514f081 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -133,6 +133,8 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Add directory to Python path
+        run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
       - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Install dev dependencies

From 56b00adb009fd6535e66ff5f9cf0eabbfc76293f Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 20 May 2024 19:15:17 -0600
Subject: [PATCH 12/60] feat(tests): add integration test to CI.

---
 .github/workflows/run_tests.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index b94514f081..ea74393ec2 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -139,5 +139,9 @@ jobs:
         run: pip install -r requirements.txt
       - name: Install dev dependencies
         run: pip install -r requirements-dev.txt
+      - name: Set up cache directory
+        run: |
+          mkdir -p cache
+          echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
       - name: Run tests with pytest
         run: pytest tests_integration/

From 742eb56616fdd8673df9d7f21acafc4792e1c604 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:06:44 -0600
Subject: [PATCH 13/60] feat(tests): add pytest.ini for integration

---
 requirements-dev.txt         | 3 ++-
 tests_integration/base.py    | 1 +
 tests_integration/pytest.ini | 6 ++++++
 3 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 tests_integration/pytest.ini

diff --git a/requirements-dev.txt b/requirements-dev.txt
index e9fdfc8867..d769cde648 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,6 +1,7 @@
 black==24.2.0
 pre-commit==3.7.0
-pytest==6.2.5
+pytest==8.2.1
+pytest-env==1.1.3
 pytest-mock==3.12.0
 ruff==0.3.0
 torch==2.2.1
diff --git a/tests_integration/base.py b/tests_integration/base.py
index ef079d9e38..abdc81f4d0 100644
--- a/tests_integration/base.py
+++ b/tests_integration/base.py
@@ -7,6 +7,7 @@
 class BaseIntegrationTestWithCache:
     @pytest.fixture(autouse=True)
     def setup(self) -> None:
+        return
         # Base directory for all DSPy modules
         library_dir = Path(__file__).resolve().parent
         base_dir = library_dir.parent
diff --git a/tests_integration/pytest.ini b/tests_integration/pytest.ini
new file mode 100644
index 0000000000..1dcef2d8e5
--- /dev/null
+++ b/tests_integration/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+filterwarnings =
+    ignore::DeprecationWarning
+
+env =
+    DSP_NOTEBOOK_CACHEDIR=/Users/amirmehr/workspace/personal_stuff/dspy/cache

From 598f8319dbadb6bca5cc3bfc355092b020a0ae5c Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:13:01 -0600
Subject: [PATCH 14/60] feat(tests): add pytest.ini for integration

---
 tests_integration/pytest.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests_integration/pytest.ini b/tests_integration/pytest.ini
index 1dcef2d8e5..60438fab36 100644
--- a/tests_integration/pytest.ini
+++ b/tests_integration/pytest.ini
@@ -3,4 +3,4 @@ filterwarnings =
     ignore::DeprecationWarning
 
 env =
-    DSP_NOTEBOOK_CACHEDIR=/Users/amirmehr/workspace/personal_stuff/dspy/cache
+    DSP_NOTEBOOK_CACHEDIR=../cache

From 73ff317f05e095f5802aae03a3cab42a61173a39 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:22:09 -0600
Subject: [PATCH 15/60] feat(tests): change it to ci way -_-

---
 tests_integration/pytest.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests_integration/pytest.ini b/tests_integration/pytest.ini
index 60438fab36..3915f3f46f 100644
--- a/tests_integration/pytest.ini
+++ b/tests_integration/pytest.ini
@@ -3,4 +3,4 @@ filterwarnings =
     ignore::DeprecationWarning
 
 env =
-    DSP_NOTEBOOK_CACHEDIR=../cache
+    DSP_NOTEBOOK_CACHEDIR=${DSP_NOTEBOOK_CACHEDIR}

From f6a64e36d12a15e387d4fccc970a4f1227c2b10a Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:26:36 -0600
Subject: [PATCH 16/60] feat(tests): change it to ci way -_-

---
 .pre-commit-config.yaml    | 12 ++++++------
 dsp/modules/cache_utils.py | 10 +++++++---
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index dd09e96273..7aa32a1823 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,12 +5,12 @@ default_stages: [commit]
 default_install_hook_types: [pre-commit, commit-msg]
 
 repos:
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.11
-    hooks:
-      - id: ruff
-        args: [--fix]
-      - id: ruff-format
+  #   - repo: https://github.com/astral-sh/ruff-pre-commit
+  #     rev: v0.1.11
+  #     hooks:
+  #       - id: ruff
+  #         args: [--fix]
+  #       - id: ruff-format
 
   - repo: https://github.com/timothycrosley/isort
     rev: 5.12.0
diff --git a/dsp/modules/cache_utils.py b/dsp/modules/cache_utils.py
index a7e2bde2ff..913d3614c5 100644
--- a/dsp/modules/cache_utils.py
+++ b/dsp/modules/cache_utils.py
@@ -6,7 +6,7 @@
 
 from dsp.utils import dotdict
 
-cache_turn_on = os.environ.get('DSP_CACHEBOOL', 'True').lower() != 'false'
+cache_turn_on = os.environ.get("DSP_CACHEBOOL", "True").lower() != "false"
 
 
 def noop_decorator(arg=None, *noop_args, **noop_kwargs):
@@ -23,10 +23,14 @@ def wrapper(*args, **kwargs):
         return decorator
 
 
-cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), 'cachedir_joblib')
+cachedir = os.environ.get("DSP_CACHEDIR") or os.path.join(Path.home(), "cachedir_joblib")
 CacheMemory = Memory(location=cachedir, verbose=0)
 
-cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')
+cachedir2 = os.environ.get("DSP_NOTEBOOK_CACHEDIR")
+print("HELLOOOOO CACHEEEE READ MEEEEEEEEE")
+print(f"cachedir2: {cachedir2}")
+print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
+print("HELLOOOOO CACHEEEE READ MEEEEEEEEE")
 NotebookCacheMemory = dotdict()
 NotebookCacheMemory.cache = noop_decorator
 

From e593154132e102790037f64f6938f2417fe0dd11 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:31:05 -0600
Subject: [PATCH 17/60] feat(tests): change it to ci way -_-

---
 .github/workflows/run_tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index ea74393ec2..d20213269f 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -141,6 +141,7 @@ jobs:
         run: pip install -r requirements-dev.txt
       - name: Set up cache directory
         run: |
+          cd ..
           mkdir -p cache
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
       - name: Run tests with pytest

From 35c39ad5f8a5ef90ff6d34060d5147ff28f5f6c3 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:35:23 -0600
Subject: [PATCH 18/60] feat(tests): change it to ci way -_-

---
 .github/workflows/run_tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index d20213269f..eb1e42d181 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -141,6 +141,7 @@ jobs:
         run: pip install -r requirements-dev.txt
       - name: Set up cache directory
         run: |
+          ls -la
           cd ..
           mkdir -p cache
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV

From 065f2d120b500652e6d684fa3b86a70093e18d3f Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:41:37 -0600
Subject: [PATCH 19/60] feat(tests): change it to ci way -_-

---
 tests_integration/pytest.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests_integration/pytest.ini b/tests_integration/pytest.ini
index 3915f3f46f..e73e3c45ce 100644
--- a/tests_integration/pytest.ini
+++ b/tests_integration/pytest.ini
@@ -3,4 +3,4 @@ filterwarnings =
     ignore::DeprecationWarning
 
 env =
-    DSP_NOTEBOOK_CACHEDIR=${DSP_NOTEBOOK_CACHEDIR}
+    DSP_NOTEBOOK_CACHEDIR=/home/runner/work/dspy/dspy/cache

From 81851d124cbc7faf54ea2af31e250375d01f7e59 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:45:11 -0600
Subject: [PATCH 20/60] feat(tests): change it to ci way -_-

---
 .github/workflows/run_tests.yml | 2 --
 tests/pytest.ini                | 3 ---
 2 files changed, 5 deletions(-)
 delete mode 100644 tests/pytest.ini

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index eb1e42d181..ec2a2a9836 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -142,8 +142,6 @@ jobs:
       - name: Set up cache directory
         run: |
           ls -la
-          cd ..
-          mkdir -p cache
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
       - name: Run tests with pytest
         run: pytest tests_integration/
diff --git a/tests/pytest.ini b/tests/pytest.ini
deleted file mode 100644
index c24fe5bb9e..0000000000
--- a/tests/pytest.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[pytest]
-filterwarnings =
-    ignore::DeprecationWarning

From f7fa69878f836b6de7c2705ac130fa68867afb29 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:45:54 -0600
Subject: [PATCH 21/60] feat(tests): change it to ci way -_-

---
 .github/workflows/run_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index ec2a2a9836..24877cd680 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -144,4 +144,4 @@ jobs:
           ls -la
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
       - name: Run tests with pytest
-        run: pytest tests_integration/
+        run: pytest -c tests_integration/pytest.ini tests_integration/

From 3b7da8d81657a75e54a62236b482070ccf5dc5ad Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:49:27 -0600
Subject: [PATCH 22/60] feat(tests): change it to ci way -_-

---
 .github/workflows/run_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 24877cd680..2e2d7a0af3 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -141,7 +141,7 @@ jobs:
         run: pip install -r requirements-dev.txt
       - name: Set up cache directory
         run: |
-          ls -la
+          ls -la cache
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
       - name: Run tests with pytest
         run: pytest -c tests_integration/pytest.ini tests_integration/

From 11c84bb45226f0dc2936f75ad0c1608851f73d3b Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 12:49:46 -0600
Subject: [PATCH 23/60] feat(tests): change it to ci way -_-

---
 .github/workflows/run_tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 2e2d7a0af3..f1539f169e 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -141,6 +141,7 @@ jobs:
         run: pip install -r requirements-dev.txt
       - name: Set up cache directory
         run: |
+          pwd ./cache
           ls -la cache
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
       - name: Run tests with pytest

From d96c0ce78d34f171d42253bb76bf809c308b825d Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:01:50 -0600
Subject: [PATCH 24/60] feat(tests): change it to ci way -_-

---
 tests_integration/test_intro.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index ec0c8c7511..ec2d76026b 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -1,5 +1,6 @@
 from typing import Any
 
+import dspy
 from tests_integration.base import BaseIntegrationTestWithCache
 
 
@@ -9,13 +10,13 @@ def test_dspy_workflow(self) -> None:
 
         dev_example, dev_set, training_set = self.assert_dataset_loading()
 
-        self.assert_basic_qa(dev_example, dspy)
+        self.assert_basic_qa(dev_example)
 
-        self.assert_retrieval(dev_example, dspy)
+        self.assert_retrieval(dev_example)
 
-        self.assert_compilation(dev_set, dspy, training_set)
+        self.assert_compilation(dev_set, training_set)
 
-    def assert_compilation(self, devset, dspy, trainset) -> None:
+    def assert_compilation(self, devset, trainset) -> None:
         class GenerateAnswer(dspy.Signature):
             """Answer questions with short factoid answers."""
 
@@ -169,7 +170,7 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
         )
         assert compiled_baleen("How many storeys are in the castle that David Gregory inherited?") is not None
 
-    def assert_retrieval(self, dev_example, dspy) -> None:
+    def assert_retrieval(self, dev_example) -> None:
         retrieve = dspy.Retrieve(k=3)
         top_k_passages = retrieve(dev_example.question).passages
 
@@ -195,7 +196,7 @@ def assert_retrieval(self, dev_example, dspy) -> None:
         retrieved_value = retrieve("When was the first FIFA World Cup held?").passages[0]
         assert retrieved_value[:30] == "History of the FIFA World Cup "
 
-    def assert_basic_qa(self, dev_example, dspy) -> None:
+    def assert_basic_qa(self, dev_example) -> None:
         class BasicQA(dspy.Signature):
             """Answer questions with short factoid answers."""
 
@@ -268,8 +269,6 @@ def assert_dataset_loading(self) -> tuple:
         return dev_example, devset, trainset
 
     def setup_dspy(self) -> Any:
-        import dspy
-
         turbo = dspy.OpenAI(model="gpt-3.5-turbo")
         colbertv2_wiki17_abstracts = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")
         dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)

From b4f33913af5a670b27dc4cdff9f55ef5e0dd9dac Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:04:22 -0600
Subject: [PATCH 25/60] feat(tests): change it to ci way -_-

---
 .github/workflows/run_tests.yml | 65 ++++++++++++++++++++-------------
 1 file changed, 40 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index f1539f169e..6c1ddf576b 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -121,28 +121,43 @@ jobs:
       - name: Run setup.py build
         run: python setup.py build
 
-  integration_test:
-    name: Run Integration Tests
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Add directory to Python path
-        run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
-      - name: Install dependencies
-        run: pip install -r requirements.txt
-      - name: Install dev dependencies
-        run: pip install -r requirements-dev.txt
-      - name: Set up cache directory
-        run: |
-          pwd ./cache
-          ls -la cache
-          echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
-      - name: Run tests with pytest
-        run: pytest -c tests_integration/pytest.ini tests_integration/
+integration_test:
+  name: Run Integration Tests
+  runs-on: ubuntu-latest
+  strategy:
+    matrix:
+      python-version: ["3.9"]
+  steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip dependencies
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ matrix.python-version }}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.python-version }}
+          ${{ runner.os }}-pip
+
+    - name: Add directory to Python path
+      run: echo "PYTHONPATH=$(python -c 'import sys; print(':'.join(sys.path))'):$(pwd)" >> $GITHUB_ENV
+
+    - name: Install dependencies
+      run: pip install -r requirements.txt
+
+    - name: Install dev dependencies
+      run: pip install -r requirements-dev.txt
+
+    - name: Set up cache directory
+      run: |
+        mkdir -p ./cache
+        ls -la cache
+        echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
+
+    - name: Run tests with pytest
+      run: pytest -c tests_integration/pytest.ini tests_integration/

From 0098e4e74316924d7fbf6717e2c20ac06f5be1b0 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:07:52 -0600
Subject: [PATCH 26/60] feat(tests): cache file.

---
 .github/workflows/run_tests.yml | 63 +++++++++++++++------------------
 tests_integration/base.py       |  1 -
 2 files changed, 28 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 6c1ddf576b..46a1f676b3 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -121,43 +121,36 @@ jobs:
       - name: Run setup.py build
         run: python setup.py build
 
-integration_test:
-  name: Run Integration Tests
-  runs-on: ubuntu-latest
-  strategy:
-    matrix:
-      python-version: ["3.9"]
-  steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Cache pip dependencies
-      uses: actions/cache@v3
-      with:
+  integration_test:
+    name: Run Integration Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ matrix.python-version }}
         restore-keys: |
           ${{ runner.os }}-pip-${{ matrix.python-version }}
           ${{ runner.os }}-pip
-
-    - name: Add directory to Python path
-      run: echo "PYTHONPATH=$(python -c 'import sys; print(':'.join(sys.path))'):$(pwd)" >> $GITHUB_ENV
-
-    - name: Install dependencies
-      run: pip install -r requirements.txt
-
-    - name: Install dev dependencies
-      run: pip install -r requirements-dev.txt
-
-    - name: Set up cache directory
-      run: |
-        mkdir -p ./cache
-        ls -la cache
-        echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
-
-    - name: Run tests with pytest
-      run: pytest -c tests_integration/pytest.ini tests_integration/
+      - name: Add directory to Python path
+        run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Install dev dependencies
+        run: pip install -r requirements-dev.txt
+      - name: Set up cache directory
+        run: |
+          pwd ./cache
+          ls -la cache
+          echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
+      - name: Run tests with pytest
+        run: pytest -c tests_integration/pytest.ini tests_integration/
diff --git a/tests_integration/base.py b/tests_integration/base.py
index abdc81f4d0..ef079d9e38 100644
--- a/tests_integration/base.py
+++ b/tests_integration/base.py
@@ -7,7 +7,6 @@
 class BaseIntegrationTestWithCache:
     @pytest.fixture(autouse=True)
     def setup(self) -> None:
-        return
         # Base directory for all DSPy modules
         library_dir = Path(__file__).resolve().parent
         base_dir = library_dir.parent

From 8f81e6ce844f72e756b888e5cb51aabe2fda8c37 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:08:39 -0600
Subject: [PATCH 27/60] feat(tests): cache file.

---
 .github/workflows/run_tests.yml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 46a1f676b3..f1539f169e 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -133,14 +133,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Cache pip dependencies
-        uses: actions/cache@v3
-        with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ matrix.python-version }}
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}
-          ${{ runner.os }}-pip
       - name: Add directory to Python path
         run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
       - name: Install dependencies

From b2e51bbf76d7463717515ffbede46ef0cf9504b8 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:10:04 -0600
Subject: [PATCH 28/60] feat(tests): cache file.

---
 .github/workflows/run_tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index f1539f169e..5e7f4b1e4c 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -133,6 +133,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
       - name: Add directory to Python path
         run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
       - name: Install dependencies

From cac66f8efd86f128ce643a14178e060e813d7a90 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:18:19 -0600
Subject: [PATCH 29/60] feat(tests): cache file.

---
 tests_integration/base.py       | 17 -----------------
 tests_integration/test_intro.py | 18 +++++++++++++++++-
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/tests_integration/base.py b/tests_integration/base.py
index ef079d9e38..e69de29bb2 100644
--- a/tests_integration/base.py
+++ b/tests_integration/base.py
@@ -1,17 +0,0 @@
-import os
-from pathlib import Path
-
-import pytest
-
-
-class BaseIntegrationTestWithCache:
-    @pytest.fixture(autouse=True)
-    def setup(self) -> None:
-        # Base directory for all DSPy modules
-        library_dir = Path(__file__).resolve().parent
-        base_dir = library_dir.parent
-        cache_dir = str(base_dir / "cache")
-        os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
-
-        if cache_dir and not Path(cache_dir).exists():
-            Path(cache_dir).mkdir(parents=True)
diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index ec2d76026b..7c9d2a185f 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -1,7 +1,23 @@
+import os
+from pathlib import Path
 from typing import Any
 
+import pytest
+
 import dspy
-from tests_integration.base import BaseIntegrationTestWithCache
+
+
+class BaseIntegrationTestWithCache:
+    @pytest.fixture(autouse=True)
+    def setup(self) -> None:
+        # Base directory for all DSPy modules
+        library_dir = Path(__file__).resolve().parent
+        base_dir = library_dir.parent
+        cache_dir = str(base_dir / "cache")
+        os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
+
+        if cache_dir and not Path(cache_dir).exists():
+            Path(cache_dir).mkdir(parents=True)
 
 
 class TestIntroIntegration(BaseIntegrationTestWithCache):

From a7c229e4bc02c55bcec0a490a2ae267f33277f53 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:18:43 -0600
Subject: [PATCH 30/60] feat(tests): cache file.

---
 .github/workflows/run_tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 5e7f4b1e4c..967dc5454b 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -142,8 +142,9 @@ jobs:
         run: pip install -r requirements-dev.txt
       - name: Set up cache directory
         run: |
-          pwd ./cache
-          ls -la cache
+          cd ./cache
+          pwd
+          ls -la
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
       - name: Run tests with pytest
         run: pytest -c tests_integration/pytest.ini tests_integration/

From 4ad4b9ff1197295bc18bc0ba443ee601f43131e1 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:24:39 -0600
Subject: [PATCH 31/60] feat(tests): cache file.

---
 .github/workflows/run_tests.yml | 2 --
 requirements.txt                | 8 ++++++++
 tests_integration/test_intro.py | 1 +
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 967dc5454b..b8b755d00e 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -138,8 +138,6 @@ jobs:
         run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
       - name: Install dependencies
         run: pip install -r requirements.txt
-      - name: Install dev dependencies
-        run: pip install -r requirements-dev.txt
       - name: Set up cache directory
         run: |
           cd ./cache
diff --git a/requirements.txt b/requirements.txt
index e5569d1f3d..7696150593 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,20 @@
 backoff
+black==24.2.0
 datasets
 joblib<=1.3.2
 openai>=0.28.1,<2.0.0
 optuna
 pandas
+pre-commit==3.7.0
 pydantic~=2.0
+pytest==8.2.1
+pytest-env==1.1.3
+pytest-mock==3.12.0
 regex
 requests
+ruff==0.3.0
 structlog
+torch==2.2.1
 tqdm
+transformers==4.38.2
 ujson
diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index 7c9d2a185f..01b814a60e 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -10,6 +10,7 @@
 class BaseIntegrationTestWithCache:
     @pytest.fixture(autouse=True)
     def setup(self) -> None:
+        return
         # Base directory for all DSPy modules
         library_dir = Path(__file__).resolve().parent
         base_dir = library_dir.parent

From 29ec21f781ace7a7fdba6aaca5019a63ac84e526 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:28:50 -0600
Subject: [PATCH 32/60] feat(tests): add print for start file.

---
 tests_integration/test_intro.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index 01b814a60e..20bd0f2b92 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -23,7 +23,11 @@ def setup(self) -> None:
 
 class TestIntroIntegration(BaseIntegrationTestWithCache):
     def test_dspy_workflow(self) -> None:
-        dspy = self.setup_dspy()
+        print("HELLOOOOO CACHEEEE READ MEEEEEEEEE START FILE")
+        print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
+        print("HELLOOOOO CACHEEEE READ MEEEEEEEEE START FILE")
+
+        self.setup_dspy()
 
         dev_example, dev_set, training_set = self.assert_dataset_loading()
 

From 9de09891283a8bdd63b3e8882f2414b74cd66b2b Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:31:54 -0600
Subject: [PATCH 33/60] feat(tests): fix versions for cache.

---
 requirements.txt | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 7696150593..ec426fb7f9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,20 +1,20 @@
-backoff
+backoff==2.2.1
 black==24.2.0
-datasets
+datasets==2.19.1
 joblib<=1.3.2
 openai>=0.28.1,<2.0.0
-optuna
-pandas
+optuna==3.6.1
+pandas==2.2.2
 pre-commit==3.7.0
 pydantic~=2.0
 pytest==8.2.1
 pytest-env==1.1.3
 pytest-mock==3.12.0
-regex
-requests
+regex==2024.5.15
+requests==2.32.0
 ruff==0.3.0
-structlog
+structlog==24.1.0
 torch==2.2.1
-tqdm
+tqdm==4.66.4
 transformers==4.38.2
-ujson
+ujson==5.10.0

From abdb9337e9d70ce490ec9d6bf91a0dcf3eeb9270 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:37:11 -0600
Subject: [PATCH 34/60] feat(tests): fix versions for cache.

---
 tests_integration/test_intro.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index 20bd0f2b92..43729804d2 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -4,6 +4,21 @@
 
 import pytest
 
+print("first")
+print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
+
+library_dir = Path(__file__).resolve().parent
+base_dir = library_dir.parent
+cache_dir = str(base_dir / "cache")
+os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
+
+if cache_dir and not Path(cache_dir).exists():
+    Path(cache_dir).mkdir(parents=True)
+
+print("second")
+
+print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
+
 import dspy
 
 
@@ -293,4 +308,3 @@ def setup_dspy(self) -> Any:
         turbo = dspy.OpenAI(model="gpt-3.5-turbo")
         colbertv2_wiki17_abstracts = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")
         dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)
-        return dspy

From c5e1e2899cd0ed6a6eed45cc6374ce1f68b114f3 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:39:26 -0600
Subject: [PATCH 35/60] feat(tests): fix cache in the ci.

---
 requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index ec426fb7f9..5a7a371bbc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,11 +2,11 @@ backoff==2.2.1
 black==24.2.0
 datasets==2.19.1
 joblib<=1.3.2
-openai>=0.28.1,<2.0.0
+openai==0.28.1
 optuna==3.6.1
 pandas==2.2.2
 pre-commit==3.7.0
-pydantic~=2.0
+pydantic==2.7.1
 pytest==8.2.1
 pytest-env==1.1.3
 pytest-mock==3.12.0

From 87dd340e9fb2e5837801314b0faf2b229d57d632 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:39:33 -0600
Subject: [PATCH 36/60] feat(tests): fix cache in the ci.

---
 .github/workflows/run_tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index b8b755d00e..078395250a 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -134,10 +134,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
+        run: pip install -r requirements.txt
       - name: Add directory to Python path
         run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
-      - name: Install dependencies
-        run: pip install -r requirements.txt
       - name: Set up cache directory
         run: |
           cd ./cache

From f14c91d79c50cefd1c43cfe39240a7fe7cfbaa6c Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 13:41:46 -0600
Subject: [PATCH 37/60] feat(tests): fix cache in the ci.

---
 .github/workflows/run_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 078395250a..cb7576074d 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -134,7 +134,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
-        run: pip install -r requirements.txt
+      - run: pip install -r requirements.txt
       - name: Add directory to Python path
         run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
       - name: Set up cache directory

From 981119889ca8c49753c0eef3b814beaac3fee3c4 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 15:55:21 -0600
Subject: [PATCH 38/60] feat(tests): remove top import.

---
 tests_integration/test_intro.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index 43729804d2..b00e896f3e 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -4,22 +4,23 @@
 
 import pytest
 
-print("first")
-print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
-
-library_dir = Path(__file__).resolve().parent
-base_dir = library_dir.parent
-cache_dir = str(base_dir / "cache")
-os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
-
-if cache_dir and not Path(cache_dir).exists():
-    Path(cache_dir).mkdir(parents=True)
-
-print("second")
+import dspy
 
-print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
+# print("first")
+# print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
+#
+# library_dir = Path(__file__).resolve().parent
+# base_dir = library_dir.parent
+# cache_dir = str(base_dir / "cache")
+# os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
+#
+# if cache_dir and not Path(cache_dir).exists():
+#     Path(cache_dir).mkdir(parents=True)
+#
+# print("second")
+#
+# print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
 
-import dspy
 
 
 class BaseIntegrationTestWithCache:

From 9c2652de726053feb9611a0d67af91a75410adca Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:03:21 -0600
Subject: [PATCH 39/60] feat(tests): add new pytest-ci.ini

---
 .github/workflows/run_tests.yml | 2 +-
 tests_integration/pytest-ci.ini | 6 ++++++
 tests_integration/pytest.ini    | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)
 create mode 100644 tests_integration/pytest-ci.ini

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index cb7576074d..537c12736c 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -144,4 +144,4 @@ jobs:
           ls -la
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
       - name: Run tests with pytest
-        run: pytest -c tests_integration/pytest.ini tests_integration/
+        run: pytest -c tests_integration/pytest-ci.ini tests_integration/
diff --git a/tests_integration/pytest-ci.ini b/tests_integration/pytest-ci.ini
new file mode 100644
index 0000000000..3915f3f46f
--- /dev/null
+++ b/tests_integration/pytest-ci.ini
@@ -0,0 +1,6 @@
+[pytest]
+filterwarnings =
+    ignore::DeprecationWarning
+
+env =
+    DSP_NOTEBOOK_CACHEDIR=${DSP_NOTEBOOK_CACHEDIR}
diff --git a/tests_integration/pytest.ini b/tests_integration/pytest.ini
index e73e3c45ce..60438fab36 100644
--- a/tests_integration/pytest.ini
+++ b/tests_integration/pytest.ini
@@ -3,4 +3,4 @@ filterwarnings =
     ignore::DeprecationWarning
 
 env =
-    DSP_NOTEBOOK_CACHEDIR=/home/runner/work/dspy/dspy/cache
+    DSP_NOTEBOOK_CACHEDIR=../cache

From f5573c57353423becaabbff9e6a5ecb8287fccf6 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:06:51 -0600
Subject: [PATCH 40/60] feat(tests): fix run_tests.yml

---
 .github/workflows/run_tests.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 537c12736c..b5d898df87 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -140,8 +140,6 @@ jobs:
       - name: Set up cache directory
         run: |
           cd ./cache
-          pwd
-          ls -la
-          echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)/cache" >> $GITHUB_ENV
+          echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)" >> $GITHUB_ENV
       - name: Run tests with pytest
         run: pytest -c tests_integration/pytest-ci.ini tests_integration/

From d6a36f934f1a947dcff941d259fa82ba4af4ffa2 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:10:15 -0600
Subject: [PATCH 41/60] feat(tests): fix run_tests.yml

---
 .github/workflows/run_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index b5d898df87..4ca9d64874 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -142,4 +142,4 @@ jobs:
           cd ./cache
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)" >> $GITHUB_ENV
       - name: Run tests with pytest
-        run: pytest -c tests_integration/pytest-ci.ini tests_integration/
+        run: pytest -c tests_integration/pytest.ini tests_integration/

From 5d994d4f71f3a58a72db714d342c1d37feb69260 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:13:16 -0600
Subject: [PATCH 42/60] feat(tests): fix run_tests.yml

---
 tests_integration/pytest.ini    |  2 +-
 tests_integration/test_intro.py | 82 +++++++++++++--------------------
 2 files changed, 34 insertions(+), 50 deletions(-)

diff --git a/tests_integration/pytest.ini b/tests_integration/pytest.ini
index 60438fab36..250e022ee5 100644
--- a/tests_integration/pytest.ini
+++ b/tests_integration/pytest.ini
@@ -3,4 +3,4 @@ filterwarnings =
     ignore::DeprecationWarning
 
 env =
-    DSP_NOTEBOOK_CACHEDIR=../cache
+    DSP_NOTEBOOK_CACHEDIR=./cache
diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index b00e896f3e..08b657eeb3 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -6,22 +6,6 @@
 
 import dspy
 
-# print("first")
-# print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
-#
-# library_dir = Path(__file__).resolve().parent
-# base_dir = library_dir.parent
-# cache_dir = str(base_dir / "cache")
-# os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
-#
-# if cache_dir and not Path(cache_dir).exists():
-#     Path(cache_dir).mkdir(parents=True)
-#
-# print("second")
-#
-# print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
-
-
 
 class BaseIntegrationTestWithCache:
     @pytest.fixture(autouse=True)
@@ -95,8 +79,8 @@ def validate_context_and_answer(example, pred, trace=None):  # noqa
         assert f"Question: {my_question}" == "Question: What castle did David Gregory inherit?"
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: Kinnairdy Castle"
         assert (
-            f"Retrieved Contexts (truncated): {[c[:10] + '...' for c in pred.context]}"
-            == "Retrieved Contexts (truncated): ['David Greg...', 'Gregory Ta...', 'David Greg...']"
+                f"Retrieved Contexts (truncated): {[c[:10] + '...' for c in pred.context]}"
+                == "Retrieved Contexts (truncated): ['David Greg...', 'Gregory Ta...', 'David Greg...']"
         )
 
         # Verify compiled model's parameters
@@ -155,7 +139,7 @@ def forward(self, question):
 
         # Assertions to verify the SimplifiedBaleen model
         assert (
-            f"Question: {my_question}" == "Question: How many storeys are in the castle that David Gregory inherited?"
+                f"Question: {my_question}" == "Question: How many storeys are in the castle that David Gregory inherited?"
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: five"
 
@@ -180,7 +164,7 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
             if max([len(h) for h in hops]) > 100:
                 return False
             if any(
-                dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))
+                    dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))
             ):
                 return False
 
@@ -198,12 +182,12 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
         # Assertions for the retrieval scores
         assert f"## Retrieval Score for RAG: {compiled_rag_retrieval_score}" == "## Retrieval Score for RAG: 26.0"
         assert (
-            f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}"
-            == "## Retrieval Score for uncompiled Baleen: 36.0"
+                f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}"
+                == "## Retrieval Score for uncompiled Baleen: 36.0"
         )
         assert (
-            f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}"
-            == "## Retrieval Score for compiled Baleen: 60.0"
+                f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}"
+                == "## Retrieval Score for compiled Baleen: 60.0"
         )
         assert compiled_baleen("How many storeys are in the castle that David Gregory inherited?") is not None
 
@@ -214,19 +198,19 @@ def assert_retrieval(self, dev_example) -> None:
         # Assertions to verify the retrieval functionality
         assert retrieve.k == 3
         assert (
-            dev_example.question
-            == "What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+                dev_example.question
+                == "What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert (
-            top_k_passages[0]
-            == "Restaurant: Impossible | Restaurant: Impossible is an American reality television series, featuring "
-            "chef and restaurateur Robert Irvine, that aired on Food Network from 2011 to 2016."
+                top_k_passages[0]
+                == "Restaurant: Impossible | Restaurant: Impossible is an American reality television series, featuring "
+                   "chef and restaurateur Robert Irvine, that aired on Food Network from 2011 to 2016."
         )
         assert (
-            top_k_passages[1]
-            == "Jean Joho | Jean Joho is a French-American chef and restaurateur. He is chef/proprietor of Everest in "
-            "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
-            "Restaurant in Las Vegas, and Brasserie JO in Boston."
+                top_k_passages[1]
+                == "Jean Joho | Jean Joho is a French-American chef and restaurateur. He is chef/proprietor of Everest in "
+                   "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
+                   "Restaurant in Las Vegas, and Brasserie JO in Boston."
         )
         assert top_k_passages[2][:30] == "List of Restaurant: Impossible"
 
@@ -247,8 +231,8 @@ class BasicQA(dspy.Signature):
 
         # Assertions to verify the basic QA functionality
         assert (
-            f"Question: {dev_example.question}"
-            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+                f"Question: {dev_example.question}"
+                == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: American"
 
@@ -259,12 +243,12 @@ class BasicQA(dspy.Signature):
 
         # Assertions to verify the chain of thought functionality
         assert (
-            f"Question: {dev_example.question}"
-            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+                f"Question: {dev_example.question}"
+                == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert (
-            f"Thought: {pred.rationale.split('.', 1)[1].strip()}"
-            == "Thought: We know that the chef and restaurateur featured in Restaurant: Impossible is Robert Irvine."
+                f"Thought: {pred.rationale.split('.', 1)[1].strip()}"
+                == "Thought: We know that the chef and restaurateur featured in Restaurant: Impossible is Robert Irvine."
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: British"
 
@@ -281,27 +265,27 @@ def assert_dataset_loading(self) -> tuple:
 
         # Assertions to verify the dataset loading
         assert (
-            f"Question: {train_example.question}"
-            == "Question: At My Window was released by which American singer-songwriter?"
+                f"Question: {train_example.question}"
+                == "Question: At My Window was released by which American singer-songwriter?"
         )
         assert f"Answer: {train_example.answer}" == "Answer: John Townes Van Zandt"
         dev_example = devset[18]
         assert (
-            f"Question: {dev_example.question}"
-            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+                f"Question: {dev_example.question}"
+                == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert f"Answer: {dev_example.answer}" == "Answer: English"
         assert "Restaurant: Impossible" in list(dev_example.gold_titles)
         assert "Robert Irvine" in list(dev_example.gold_titles)
         assert (
-            f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys "
-            f"{train_example.labels().keys()}"
-            == "For this dataset, training examples have input keys ['question'] and label keys ['answer']"
+                f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys "
+                f"{train_example.labels().keys()}"
+                == "For this dataset, training examples have input keys ['question'] and label keys ['answer']"
         )
         assert (
-            f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys "
-            f"{dev_example.labels().keys()}"
-            == "For this dataset, dev examples have input keys ['question'] and label keys ['answer', 'gold_titles']"
+                f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys "
+                f"{dev_example.labels().keys()}"
+                == "For this dataset, dev examples have input keys ['question'] and label keys ['answer', 'gold_titles']"
         )
         return dev_example, devset, trainset
 

From a6f1395939015f508b39eaab387eb21f604f888a Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:17:39 -0600
Subject: [PATCH 43/60] feat(tests): remove pytest-ci.ini

---
 tests_integration/pytest-ci.ini | 6 ------
 1 file changed, 6 deletions(-)
 delete mode 100644 tests_integration/pytest-ci.ini

diff --git a/tests_integration/pytest-ci.ini b/tests_integration/pytest-ci.ini
deleted file mode 100644
index 3915f3f46f..0000000000
--- a/tests_integration/pytest-ci.ini
+++ /dev/null
@@ -1,6 +0,0 @@
-[pytest]
-filterwarnings =
-    ignore::DeprecationWarning
-
-env =
-    DSP_NOTEBOOK_CACHEDIR=${DSP_NOTEBOOK_CACHEDIR}

From 5f31aa226ee66c005d4fadaa91f7cfb111181398 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:18:14 -0600
Subject: [PATCH 44/60] feat(tests): split requirements and requirements-dev

---
 .github/workflows/run_tests.yml | 1 +
 requirements.txt                | 8 --------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 4ca9d64874..b947fd3e8e 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -135,6 +135,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
       - run: pip install -r requirements.txt
+      - run: pip install -r requirements-dev.txt
       - name: Add directory to Python path
         run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
       - name: Set up cache directory
diff --git a/requirements.txt b/requirements.txt
index 5a7a371bbc..05d762f68f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,20 +1,12 @@
 backoff==2.2.1
-black==24.2.0
 datasets==2.19.1
 joblib<=1.3.2
 openai==0.28.1
 optuna==3.6.1
 pandas==2.2.2
-pre-commit==3.7.0
 pydantic==2.7.1
-pytest==8.2.1
-pytest-env==1.1.3
-pytest-mock==3.12.0
 regex==2024.5.15
 requests==2.32.0
-ruff==0.3.0
 structlog==24.1.0
-torch==2.2.1
 tqdm==4.66.4
-transformers==4.38.2
 ujson==5.10.0

From 53fb2a5866764b44383a88af2a2898bb26aaafcf Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:21:16 -0600
Subject: [PATCH 45/60] feat(tests): revert cache

---
 dsp/modules/cache_utils.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/dsp/modules/cache_utils.py b/dsp/modules/cache_utils.py
index 913d3614c5..739e5b4c1a 100644
--- a/dsp/modules/cache_utils.py
+++ b/dsp/modules/cache_utils.py
@@ -27,10 +27,7 @@ def wrapper(*args, **kwargs):
 CacheMemory = Memory(location=cachedir, verbose=0)
 
 cachedir2 = os.environ.get("DSP_NOTEBOOK_CACHEDIR")
-print("HELLOOOOO CACHEEEE READ MEEEEEEEEE")
-print(f"cachedir2: {cachedir2}")
-print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
-print("HELLOOOOO CACHEEEE READ MEEEEEEEEE")
+
 NotebookCacheMemory = dotdict()
 NotebookCacheMemory.cache = noop_decorator
 

From 9a847f53c3a06c5d28cda333b61f4681baf73253 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:22:16 -0600
Subject: [PATCH 46/60] feat(tests): revert cache

---
 dsp/modules/cache_utils.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/dsp/modules/cache_utils.py b/dsp/modules/cache_utils.py
index 739e5b4c1a..a7e2bde2ff 100644
--- a/dsp/modules/cache_utils.py
+++ b/dsp/modules/cache_utils.py
@@ -6,7 +6,7 @@
 
 from dsp.utils import dotdict
 
-cache_turn_on = os.environ.get("DSP_CACHEBOOL", "True").lower() != "false"
+cache_turn_on = os.environ.get('DSP_CACHEBOOL', 'True').lower() != 'false'
 
 
 def noop_decorator(arg=None, *noop_args, **noop_kwargs):
@@ -23,11 +23,10 @@ def wrapper(*args, **kwargs):
         return decorator
 
 
-cachedir = os.environ.get("DSP_CACHEDIR") or os.path.join(Path.home(), "cachedir_joblib")
+cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), 'cachedir_joblib')
 CacheMemory = Memory(location=cachedir, verbose=0)
 
-cachedir2 = os.environ.get("DSP_NOTEBOOK_CACHEDIR")
-
+cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')
 NotebookCacheMemory = dotdict()
 NotebookCacheMemory.cache = noop_decorator
 

From a11e80ee7542e89adeeaa9c1ff99d753bc2ca8f0 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:23:51 -0600
Subject: [PATCH 47/60] feat(tests): revert pre-commit.

---
 .pre-commit-config.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7aa32a1823..dd09e96273 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,12 +5,12 @@ default_stages: [commit]
 default_install_hook_types: [pre-commit, commit-msg]
 
 repos:
-  #   - repo: https://github.com/astral-sh/ruff-pre-commit
-  #     rev: v0.1.11
-  #     hooks:
-  #       - id: ruff
-  #         args: [--fix]
-  #       - id: ruff-format
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.11
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
 
   - repo: https://github.com/timothycrosley/isort
     rev: 5.12.0

From c1aa768743e825956a5fae962494e8050680b530 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:34:51 -0600
Subject: [PATCH 48/60] feat(tests): clean up.

---
 requirements.txt                | 22 +++++------
 tests_integration/test_intro.py | 70 ++++++++++++++++-----------------
 2 files changed, 44 insertions(+), 48 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 05d762f68f..e5569d1f3d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,12 @@
-backoff==2.2.1
-datasets==2.19.1
+backoff
+datasets
 joblib<=1.3.2
-openai==0.28.1
-optuna==3.6.1
-pandas==2.2.2
-pydantic==2.7.1
-regex==2024.5.15
-requests==2.32.0
-structlog==24.1.0
-tqdm==4.66.4
-ujson==5.10.0
+openai>=0.28.1,<2.0.0
+optuna
+pandas
+pydantic~=2.0
+regex
+requests
+structlog
+tqdm
+ujson
diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index 08b657eeb3..71b19c138a 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -23,10 +23,6 @@ def setup(self) -> None:
 
 class TestIntroIntegration(BaseIntegrationTestWithCache):
     def test_dspy_workflow(self) -> None:
-        print("HELLOOOOO CACHEEEE READ MEEEEEEEEE START FILE")
-        print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
-        print("HELLOOOOO CACHEEEE READ MEEEEEEEEE START FILE")
-
         self.setup_dspy()
 
         dev_example, dev_set, training_set = self.assert_dataset_loading()
@@ -79,8 +75,8 @@ def validate_context_and_answer(example, pred, trace=None):  # noqa
         assert f"Question: {my_question}" == "Question: What castle did David Gregory inherit?"
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: Kinnairdy Castle"
         assert (
-                f"Retrieved Contexts (truncated): {[c[:10] + '...' for c in pred.context]}"
-                == "Retrieved Contexts (truncated): ['David Greg...', 'Gregory Ta...', 'David Greg...']"
+            f"Retrieved Contexts (truncated): {[c[:10] + '...' for c in pred.context]}"
+            == "Retrieved Contexts (truncated): ['David Greg...', 'Gregory Ta...', 'David Greg...']"
         )
 
         # Verify compiled model's parameters
@@ -139,7 +135,7 @@ def forward(self, question):
 
         # Assertions to verify the SimplifiedBaleen model
         assert (
-                f"Question: {my_question}" == "Question: How many storeys are in the castle that David Gregory inherited?"
+            f"Question: {my_question}" == "Question: How many storeys are in the castle that David Gregory inherited?"
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: five"
 
@@ -164,7 +160,7 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
             if max([len(h) for h in hops]) > 100:
                 return False
             if any(
-                    dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))
+                dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))
             ):
                 return False
 
@@ -182,12 +178,12 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
         # Assertions for the retrieval scores
         assert f"## Retrieval Score for RAG: {compiled_rag_retrieval_score}" == "## Retrieval Score for RAG: 26.0"
         assert (
-                f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}"
-                == "## Retrieval Score for uncompiled Baleen: 36.0"
+            f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}"
+            == "## Retrieval Score for uncompiled Baleen: 36.0"
         )
         assert (
-                f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}"
-                == "## Retrieval Score for compiled Baleen: 60.0"
+            f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}"
+            == "## Retrieval Score for compiled Baleen: 60.0"
         )
         assert compiled_baleen("How many storeys are in the castle that David Gregory inherited?") is not None
 
@@ -198,19 +194,19 @@ def assert_retrieval(self, dev_example) -> None:
         # Assertions to verify the retrieval functionality
         assert retrieve.k == 3
         assert (
-                dev_example.question
-                == "What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+            dev_example.question
+            == "What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert (
-                top_k_passages[0]
-                == "Restaurant: Impossible | Restaurant: Impossible is an American reality television series, featuring "
-                   "chef and restaurateur Robert Irvine, that aired on Food Network from 2011 to 2016."
+            top_k_passages[0]
+            == "Restaurant: Impossible | Restaurant: Impossible is an American reality television series, featuring "
+            "chef and restaurateur Robert Irvine, that aired on Food Network from 2011 to 2016."
         )
         assert (
-                top_k_passages[1]
-                == "Jean Joho | Jean Joho is a French-American chef and restaurateur. He is chef/proprietor of Everest in "
-                   "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
-                   "Restaurant in Las Vegas, and Brasserie JO in Boston."
+            top_k_passages[1]
+            == "Jean Joho | Jean Joho is a French-American chef and restaurateur. He is chef/proprietor of Everest in "
+            "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
+            "Restaurant in Las Vegas, and Brasserie JO in Boston."
         )
         assert top_k_passages[2][:30] == "List of Restaurant: Impossible"
 
@@ -231,8 +227,8 @@ class BasicQA(dspy.Signature):
 
         # Assertions to verify the basic QA functionality
         assert (
-                f"Question: {dev_example.question}"
-                == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+            f"Question: {dev_example.question}"
+            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: American"
 
@@ -243,12 +239,12 @@ class BasicQA(dspy.Signature):
 
         # Assertions to verify the chain of thought functionality
         assert (
-                f"Question: {dev_example.question}"
-                == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+            f"Question: {dev_example.question}"
+            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert (
-                f"Thought: {pred.rationale.split('.', 1)[1].strip()}"
-                == "Thought: We know that the chef and restaurateur featured in Restaurant: Impossible is Robert Irvine."
+            f"Thought: {pred.rationale.split('.', 1)[1].strip()}"
+            == "Thought: We know that the chef and restaurateur featured in Restaurant: Impossible is Robert Irvine."
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: British"
 
@@ -265,27 +261,27 @@ def assert_dataset_loading(self) -> tuple:
 
         # Assertions to verify the dataset loading
         assert (
-                f"Question: {train_example.question}"
-                == "Question: At My Window was released by which American singer-songwriter?"
+            f"Question: {train_example.question}"
+            == "Question: At My Window was released by which American singer-songwriter?"
         )
         assert f"Answer: {train_example.answer}" == "Answer: John Townes Van Zandt"
         dev_example = devset[18]
         assert (
-                f"Question: {dev_example.question}"
-                == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+            f"Question: {dev_example.question}"
+            == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert f"Answer: {dev_example.answer}" == "Answer: English"
         assert "Restaurant: Impossible" in list(dev_example.gold_titles)
         assert "Robert Irvine" in list(dev_example.gold_titles)
         assert (
-                f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys "
-                f"{train_example.labels().keys()}"
-                == "For this dataset, training examples have input keys ['question'] and label keys ['answer']"
+            f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys "
+            f"{train_example.labels().keys()}"
+            == "For this dataset, training examples have input keys ['question'] and label keys ['answer']"
         )
         assert (
-                f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys "
-                f"{dev_example.labels().keys()}"
-                == "For this dataset, dev examples have input keys ['question'] and label keys ['answer', 'gold_titles']"
+            f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys "
+            f"{dev_example.labels().keys()}"
+            == "For this dataset, dev examples have input keys ['question'] and label keys ['answer', 'gold_titles']"
         )
         return dev_example, devset, trainset
 

From f0bd17db427535ee50e6cf4328c697e1192fe412 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:38:05 -0600
Subject: [PATCH 49/60] feat(tests): show ruff errors.

---
 .github/workflows/run_tests.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index b947fd3e8e..8bfae24be4 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -23,10 +23,12 @@ jobs:
       - name: Ruff Fix Attempt
         id: ruff_fix
         uses: chartboost/ruff-action@v1
-        with:
-          args: --fix-only --exit-non-zero-on-fix
+        run: |
+          ruff check . --fix-only --exit-non-zero-on-fix | tee output.txt
         continue-on-error: true
-
+      - name: Print Ruff Fix Output
+        run: |
+          cat output.txt
       - name: Fail Workflow if Ruff Fix Failed
         if: steps.ruff_fix.outcome == 'failure'
         run: |

From c1d49c0e9c850f28df5a75618af873bac136eb89 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:39:12 -0600
Subject: [PATCH 50/60] feat(tests): revert requirements.

---
 requirements.txt | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index e5569d1f3d..05d762f68f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,12 @@
-backoff
-datasets
+backoff==2.2.1
+datasets==2.19.1
 joblib<=1.3.2
-openai>=0.28.1,<2.0.0
-optuna
-pandas
-pydantic~=2.0
-regex
-requests
-structlog
-tqdm
-ujson
+openai==0.28.1
+optuna==3.6.1
+pandas==2.2.2
+pydantic==2.7.1
+regex==2024.5.15
+requests==2.32.0
+structlog==24.1.0
+tqdm==4.66.4
+ujson==5.10.0

From 66309ee9dd541ad5d72af08f53a4589cb16c58da Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:40:14 -0600
Subject: [PATCH 51/60] feat(tests): fix ruff.

---
 .github/workflows/run_tests.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 8bfae24be4..0a57a4eb60 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -24,11 +24,8 @@ jobs:
         id: ruff_fix
         uses: chartboost/ruff-action@v1
         run: |
-          ruff check . --fix-only --exit-non-zero-on-fix | tee output.txt
+          ruff check . --fix-only --exit-non-zero-on-fix
         continue-on-error: true
-      - name: Print Ruff Fix Output
-        run: |
-          cat output.txt
       - name: Fail Workflow if Ruff Fix Failed
         if: steps.ruff_fix.outcome == 'failure'
         run: |

From 1be9a20fab1eaa7a779fb1505954fe576d3f5147 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:41:15 -0600
Subject: [PATCH 52/60] feat(tests): fix ruff.

---
 .github/workflows/run_tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 0a57a4eb60..9ef5e28505 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -23,8 +23,8 @@ jobs:
       - name: Ruff Fix Attempt
         id: ruff_fix
         uses: chartboost/ruff-action@v1
-        run: |
-          ruff check . --fix-only --exit-non-zero-on-fix
+        with:
+          args: --fix-only --exit-non-zero-on-fix
         continue-on-error: true
       - name: Fail Workflow if Ruff Fix Failed
         if: steps.ruff_fix.outcome == 'failure'

From 01b671dedafe47918b70e5b8e5e4fc4727d0ee7b Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:43:55 -0600
Subject: [PATCH 53/60] feat(tests): fix ruff.

---
 .github/workflows/run_tests.yml | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 9ef5e28505..5c4df11b10 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -143,3 +143,34 @@ jobs:
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)" >> $GITHUB_ENV
       - name: Run tests with pytest
         run: pytest -c tests_integration/pytest.ini tests_integration/
+  another_fix:
+    name: Check Ruff Fix 2
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - name: Ruff Fix Attempt
+        id: ruff_fix
+        uses: chartboost/ruff-action@v1
+        with:
+          args: --fix-only --exit-non-zero-on-fix
+        continue-on-error: true
+      - name: Capture Ruff Issues
+        if: steps.ruff_fix.outcome == 'failure'
+        run: |
+          echo "Ruff fix failed. Capturing issues..."
+          ruff check . --exit-zero > ruff_issues.txt
+      - name: Echo Ruff Issues
+        if: steps.ruff_fix.outcome == 'failure'
+        run: |
+          echo "Ruff issues found:"
+          cat ruff_issues.txt
+      - name: Fail Workflow if Ruff Fix Failed
+        if: steps.ruff_fix.outcome == 'failure'
+        run: |
+          echo "Ruff fix failed, failing the workflow."
+          echo "Please run 'ruff check . --fix-only' locally and push the changes."
+          exit 1

From 26f2ffd00c8144ce2c97cb2e8326db547e7c6b1c Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:47:28 -0600
Subject: [PATCH 54/60] feat(tests): fix ruff.

---
 .github/workflows/run_tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 5c4df11b10..57f51f2b16 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -152,6 +152,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
+      - name: Install Ruff
+        run: |
+          python -m pip install ruff
       - name: Ruff Fix Attempt
         id: ruff_fix
         uses: chartboost/ruff-action@v1

From b42e8bf0da8ce9fb6eab213ba62efdda1e538dc3 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:49:20 -0600
Subject: [PATCH 55/60] feat(tests): fix ruff.

---
 .github/workflows/run_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 57f51f2b16..49dc04fb0c 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -165,7 +165,7 @@ jobs:
         if: steps.ruff_fix.outcome == 'failure'
         run: |
           echo "Ruff fix failed. Capturing issues..."
-          ruff check . --exit-zero > ruff_issues.txt
+          ruff check . --fix-only --exit-non-zero-on-fix > ruff_issues.txt
       - name: Echo Ruff Issues
         if: steps.ruff_fix.outcome == 'failure'
         run: |

From c9e687e9d7b63bb29ec29f4763a68a64155d7855 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 16:56:53 -0600
Subject: [PATCH 56/60] feat(tests): cleanup.

---
 tests_integration/test_intro.py | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index 71b19c138a..ad23d2dbb3 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -1,27 +1,9 @@
-import os
-from pathlib import Path
 from typing import Any
 
-import pytest
-
 import dspy
 
 
-class BaseIntegrationTestWithCache:
-    @pytest.fixture(autouse=True)
-    def setup(self) -> None:
-        return
-        # Base directory for all DSPy modules
-        library_dir = Path(__file__).resolve().parent
-        base_dir = library_dir.parent
-        cache_dir = str(base_dir / "cache")
-        os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
-
-        if cache_dir and not Path(cache_dir).exists():
-            Path(cache_dir).mkdir(parents=True)
-
-
-class TestIntroIntegration(BaseIntegrationTestWithCache):
+class TestIntroIntegration:
     def test_dspy_workflow(self) -> None:
         self.setup_dspy()
 

From 1c65f3c40745de44411cffff2322f6a1534567aa Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 17:24:09 -0600
Subject: [PATCH 57/60] feat(tests): remove second ruff.

---
 .github/workflows/run_tests.yml | 34 ---------------------------------
 1 file changed, 34 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 49dc04fb0c..9ef5e28505 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -143,37 +143,3 @@ jobs:
           echo "DSP_NOTEBOOK_CACHEDIR=$(pwd)" >> $GITHUB_ENV
       - name: Run tests with pytest
         run: pytest -c tests_integration/pytest.ini tests_integration/
-  another_fix:
-    name: Check Ruff Fix 2
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-      - name: Install Ruff
-        run: |
-          python -m pip install ruff
-      - name: Ruff Fix Attempt
-        id: ruff_fix
-        uses: chartboost/ruff-action@v1
-        with:
-          args: --fix-only --exit-non-zero-on-fix
-        continue-on-error: true
-      - name: Capture Ruff Issues
-        if: steps.ruff_fix.outcome == 'failure'
-        run: |
-          echo "Ruff fix failed. Capturing issues..."
-          ruff check . --fix-only --exit-non-zero-on-fix > ruff_issues.txt
-      - name: Echo Ruff Issues
-        if: steps.ruff_fix.outcome == 'failure'
-        run: |
-          echo "Ruff issues found:"
-          cat ruff_issues.txt
-      - name: Fail Workflow if Ruff Fix Failed
-        if: steps.ruff_fix.outcome == 'failure'
-        run: |
-          echo "Ruff fix failed, failing the workflow."
-          echo "Please run 'ruff check . --fix-only' locally and push the changes."
-          exit 1

From e06883cff49014fbee30e3e6ca2fe8e2c8ab64ff Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Sat, 25 May 2024 22:52:57 -0600
Subject: [PATCH 58/60] feat(tests): arnav comments on intro integration

---
 tests_integration/test_intro.py | 96 ++++++++++++++++++++++++++++++---
 1 file changed, 90 insertions(+), 6 deletions(-)

diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index ad23d2dbb3..218f5ebac9 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -62,9 +62,73 @@ def validate_context_and_answer(example, pred, trace=None):  # noqa
         )
 
         # Verify compiled model's parameters
+        named_predictors_dict = {}
         for name, parameter in compiled_rag.named_predictors():
-            assert name is not None
-            assert parameter.demos[0] is not None
+            named_predictors_dict[name] = {
+                "context": parameter.demos[0].context,
+                "answer": parameter.demos[0].answer,
+                "question": parameter.demos[0].question,
+                "rationale": parameter.demos[0].rationale,
+            }
+
+        assert named_predictors_dict == {
+            "generate_answer": {
+                "context": [
+                    "Tae Kwon Do Times | Tae Kwon Do Times is a magazine devoted to the martial art of "
+                    "taekwondo, and is published in the United States of America. While the title "
+                    "suggests that it focuses on taekwondo exclusively, the magazine also covers other "
+                    'Korean martial arts. "Tae Kwon Do Times" has published articles by a wide range of '
+                    "authors, including He-Young Kimm, Thomas Kurz, Scott Shaw, and Mark Van "
+                    "Schuyver.",
+                    "Kwon Tae-man | Kwon Tae-man (born 1941) was an early Korean hapkido "
+                    "practitioner and a pioneer of the art, first in Korea and then in the "
+                    "United States. He formed one of the earliest dojang's for hapkido in "
+                    "the United States in Torrance, California, and has been featured in "
+                    "many magazine articles promoting the art.",
+                    "Hee Il Cho | Cho Hee Il "
+                    "(born October 13, "
+                    "1940) is a prominent "
+                    "Korean-American master "
+                    "of taekwondo, "
+                    "holding the rank of 9th "
+                    '"dan" in the martial '
+                    "art. He has written 11 "
+                    "martial art books, "
+                    "produced 70 martial art "
+                    "training videos, "
+                    "and has appeared on more "
+                    "than 70 martial arts "
+                    "magazine covers. Cho won "
+                    "several national and "
+                    "international "
+                    "competitions as a "
+                    "taekwondo competitor, "
+                    "and has appeared in "
+                    "several films, "
+                    'including "Fight to '
+                    'Win", "Best of the '
+                    'Best", "Bloodsport II", '
+                    'and "Bloodsport III". He '
+                    "founded the Action "
+                    "International Martial "
+                    "Arts Association (AIMAA) "
+                    "in 1980, and is its "
+                    "President. Cho is a "
+                    'member of both "Black '
+                    "Belt\" magazine's Hall "
+                    'of Fame and "Tae Kwon Do '
+                    "Times\" magazine's Hall "
+                    "of Fame.",
+                ],
+                "answer": "Tae Kwon Do Times",
+                "question": "Which magazine has published articles by Scott "
+                "Shaw, Tae Kwon Do Times or Southwest Art?",
+                "rationale": 'produce the answer. We know from the context that "Tae Kwon Do Times" is a '
+                "magazine that covers taekwondo and other Korean martial arts. It has published "
+                "articles by authors like Scott Shaw. On the other hand, there is no information "
+                "about Southwest Art magazine in the context.",
+            },
+        }
 
         from dspy.evaluate.evaluate import Evaluate
 
@@ -190,7 +254,22 @@ def assert_retrieval(self, dev_example) -> None:
             "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
             "Restaurant in Las Vegas, and Brasserie JO in Boston."
         )
-        assert top_k_passages[2][:30] == "List of Restaurant: Impossible"
+        assert top_k_passages[2] == (
+            "List of Restaurant: Impossible episodes | This is the list of the episodes for "
+            'the American cooking and reality television series "Restaurant Impossible", '
+            "produced by Food Network. The premise of the series is that within two days and "
+            "on a budget of $10,000, celebrity chef Robert Irvine renovates a failing "
+            "American restaurant with the goal of helping to restore it to profitability and "
+            "prominence. Irvine is assisted by a designer (usually Taniya Nayak, "
+            "Cheryl Torrenueva, or Lynn Keagan, but sometimes Vanessa De Leon, "
+            "Krista Watterworth, Yvette Irene, or Nicole Faccuito), along with general "
+            "contractor Tom Bury, who sometimes does double duty as both general contractor "
+            "and designer. After assessing the problems with the restaurant, Robert Irvine "
+            "typically creates a plan for the new decor, oversees the cleaning of the "
+            "restaurant, reduces the size of the menu and improves the food, develops a "
+            "promotional activity, educates the restaurant's owners, or trains the staff, "
+            "as needed by each restaurant."
+        )
 
         retrieved_value = retrieve("When was the first FIFA World Cup held?").passages[0]
         assert retrieved_value[:30] == "History of the FIFA World Cup "
@@ -213,7 +292,12 @@ class BasicQA(dspy.Signature):
             == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
         )
         assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: American"
-
+        assert (
+            self.turbo.inspect_history().strip()
+            == "Answer questions with short factoid answers.\n\n---\n\nFollow the following format.\n\nQuestion: ${"
+            "question}\nAnswer: often between 1 and 5 words\n\n---\n\nQuestion: What is the nationality of the "
+            "chef and restaurateur featured in Restaurant: Impossible?\nAnswer:\x1b[32m American\x1b[0m"
+        )
         # Define the predictor with chain of thought
         generate_answer_with_chain_of_thought = dspy.ChainOfThought(BasicQA)
         # Call the predictor on the same input
@@ -268,6 +352,6 @@ def assert_dataset_loading(self) -> tuple:
         return dev_example, devset, trainset
 
     def setup_dspy(self) -> Any:
-        turbo = dspy.OpenAI(model="gpt-3.5-turbo")
+        self.turbo = dspy.OpenAI(model="gpt-3.5-turbo")
         colbertv2_wiki17_abstracts = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")
-        dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)
+        dspy.settings.configure(lm=self.turbo, rm=colbertv2_wiki17_abstracts)

From beca1197ed5e7631ad23989fa749591bcba78c97 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 27 May 2024 17:05:44 -0600
Subject: [PATCH 59/60] feat(tests): revert requirements.txt

---
 .github/workflows/run_tests.yml |  1 +
 requirements.txt                | 22 +++++++++++-----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 9ef5e28505..06c1c6f6ea 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -135,6 +135,7 @@ jobs:
           cache: "pip"
       - run: pip install -r requirements.txt
       - run: pip install -r requirements-dev.txt
+      - run: pip install openai==0.28.1
       - name: Add directory to Python path
         run: echo "PYTHONPATH=$(python -c 'import sys; print(":".join(sys.path))'):$(pwd)" >> $GITHUB_ENV
       - name: Set up cache directory
diff --git a/requirements.txt b/requirements.txt
index 05d762f68f..e5569d1f3d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,12 @@
-backoff==2.2.1
-datasets==2.19.1
+backoff
+datasets
 joblib<=1.3.2
-openai==0.28.1
-optuna==3.6.1
-pandas==2.2.2
-pydantic==2.7.1
-regex==2024.5.15
-requests==2.32.0
-structlog==24.1.0
-tqdm==4.66.4
-ujson==5.10.0
+openai>=0.28.1,<2.0.0
+optuna
+pandas
+pydantic~=2.0
+regex
+requests
+structlog
+tqdm
+ujson

From 481e0f1073e69433b53b6b09f5b31c9948b658c0 Mon Sep 17 00:00:00 2001
From: Amir Mehr <amir.mehr@showpass.com>
Date: Mon, 27 May 2024 17:10:21 -0600
Subject: [PATCH 60/60] feat(tests): remove base.py

---
 tests_integration/base.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 tests_integration/base.py

diff --git a/tests_integration/base.py b/tests_integration/base.py
deleted file mode 100644
index e69de29bb2..0000000000