From cce629e59971f9a6d5766d18c7a62fa005d4196a Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 10:02:38 -0600 Subject: [PATCH 01/21] Add Ruff linting workflow --- .github/workflows/ruff_lint.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/ruff_lint.yml diff --git a/.github/workflows/ruff_lint.yml b/.github/workflows/ruff_lint.yml new file mode 100644 index 0000000000..67fce6a4c0 --- /dev/null +++ b/.github/workflows/ruff_lint.yml @@ -0,0 +1,24 @@ +name: Lint and Commit + +on: + push: + branches: + - main + pull_request: + +env: + POETRY_VERSION: "1.6.1" + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v2 + - run: pip install ruff + - run: | + ruff check . + ruff fix . + - uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: 'Style fixes by Ruff' \ No newline at end of file From b24abbc199d0534a4cd40afd50e2158d227ad258 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 10:35:41 -0600 Subject: [PATCH 02/21] Update GitHub Actions workflows --- .github/workflows/ruff_lint.yml | 18 +++++++++++------- .github/workflows/run_tests.yml | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ruff_lint.yml b/.github/workflows/ruff_lint.yml index 67fce6a4c0..40ef43b6cb 100644 --- a/.github/workflows/ruff_lint.yml +++ b/.github/workflows/ruff_lint.yml @@ -11,14 +11,18 @@ env: jobs: lint: + name: Lint runs-on: ubuntu-latest + permissions: + # Give the default GITHUB_TOKEN write permission to commit and push the + # added or changed files to the repository. + contents: write steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v2 - - run: pip install ruff - - run: | - ruff check . - ruff fix . - - uses: stefanzweifel/git-auto-commit-action@v4 + - uses: actions/setup-python@v5 + - uses: chartboost/ruff-action@v1 with: - commit_message: 'Style fixes by Ruff' \ No newline at end of file + args: --fix + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "Style fixes by Ruff" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4a31b86453..b346784d13 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -11,7 +11,7 @@ env: jobs: test: - name: Test + name: Run Tests runs-on: ubuntu-latest strategy: matrix: From b55b67f4c872b56e29c5140b059314e184dfdaaf Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 10:40:47 -0600 Subject: [PATCH 03/21] Make ruff apply but not fail --- .github/workflows/ruff_lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruff_lint.yml b/.github/workflows/ruff_lint.yml index 40ef43b6cb..025129e169 100644 --- a/.github/workflows/ruff_lint.yml +++ b/.github/workflows/ruff_lint.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-python@v5 - uses: chartboost/ruff-action@v1 with: - args: --fix + args: --fix-only - uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: "Style fixes by Ruff" From a09909b8d688de79ae98478f275c69654ddff9ec Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 3 Mar 2024 16:41:10 +0000 Subject: [PATCH 04/21] Style fixes by Ruff --- dspy/experimental/synthesizer.py | 12 ++++++------ dspy/experimental/synthetic_data.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dspy/experimental/synthesizer.py b/dspy/experimental/synthesizer.py index e247d35f3b..d766f2dcab 100644 --- a/dspy/experimental/synthesizer.py +++ b/dspy/experimental/synthesizer.py @@ -14,11 +14,11 @@ def format_examples(examples: List[dspy.Example]): input_keys = example.inputs().keys() label_keys = example.labels().keys() - formatted_example += f"Inputs:\n" + formatted_example += "Inputs:\n" for key in input_keys: formatted_example += f"{key}: {example[key]}\n" - formatted_example += f"Outputs:\n" + formatted_example += "Outputs:\n" for key in label_keys: formatted_example += f"{key}: {example[key]}\n" @@ -94,7 +94,7 @@ def _prepare_synthetic_data_predictors(self, input_keys: List[str], output_keys: self.generate_input_data = self.generate_input_data.insert( -1, field_name, - output_field + output_field, ) input_field = dspy.InputField( @@ -104,7 +104,7 @@ def _prepare_synthetic_data_predictors(self, input_keys: List[str], output_keys: self.generate_output_data = self.generate_output_data.insert( -1, field_name, - input_field + input_field, ) for key in tqdm(output_keys, desc="Preparing Output Fields"): @@ -123,7 +123,7 @@ def _prepare_synthetic_data_predictors(self, input_keys: List[str], output_keys: self.generate_output_data = self.generate_output_data.insert( -1, field_name, - output_field + output_field, ) return dspy.ChainOfThought(self.generate_input_data), dspy.Predict(self.generate_output_data) @@ -167,7 +167,7 @@ def export(self, data: List[dspy.Example], path: str, mode: str = None, **kwargs extention = mode or path.split(".")[-1] dataset = Dataset.from_list( - [example.toDict() for example in data] + [example.toDict() for example in data], ) if extention == "csv": diff --git a/dspy/experimental/synthetic_data.py b/dspy/experimental/synthetic_data.py index d5177fcace..b9121b98d1 100644 --- a/dspy/experimental/synthetic_data.py +++ b/dspy/experimental/synthetic_data.py @@ -55,7 +55,7 @@ def _prepare_fields(self, properties) -> dict: '__doc__': f"Generates the following outputs: {{{', '.join(properties.keys())}}}.", 'sindex': dspy.InputField(desc="a random string"), **{field_name: dspy.OutputField(desc=properties[field_name].get('description', 'No description')) - for field_name in properties.keys()} + for field_name in properties.keys()}, } # # Usage example From 70151e7cf757532522f14de9fa37a21288e29617 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 10:48:04 -0600 Subject: [PATCH 05/21] Combine Workflows --- .github/workflows/ruff_lint.yml | 28 ---------------------------- .github/workflows/run_tests.yml | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 29 deletions(-) delete mode 100644 .github/workflows/ruff_lint.yml diff --git a/.github/workflows/ruff_lint.yml b/.github/workflows/ruff_lint.yml deleted file mode 100644 index 025129e169..0000000000 --- a/.github/workflows/ruff_lint.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Lint and Commit - -on: - push: - branches: - - main - pull_request: - -env: - POETRY_VERSION: "1.6.1" - -jobs: - lint: - name: Lint - runs-on: ubuntu-latest - permissions: - # Give the default GITHUB_TOKEN write permission to commit and push the - # added or changed files to the repository. - contents: write - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v5 - - uses: chartboost/ruff-action@v1 - with: - args: --fix-only - - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: "Style fixes by Ruff" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b346784d13..83a6e26106 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -1,4 +1,4 @@ -name: Build and Run Tests +name: Lint, Commit, and Test on: push: @@ -10,8 +10,23 @@ env: POETRY_VERSION: "1.6.1" jobs: + lint: + name: Lint + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v5 + - uses: chartboost/ruff-action@v1 + with: + args: --fix-only + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "Automatic Style fixes" test: name: Run Tests + needs: lint runs-on: ubuntu-latest strategy: matrix: @@ -30,6 +45,7 @@ jobs: run: poetry run pytest tests/ build_poetry: name: Build Poetry + needs: test runs-on: ubuntu-latest strategy: matrix: @@ -52,6 +68,7 @@ jobs: run: python -c "import dsp" build_setup: name: Build Setup + needs: test runs-on: ubuntu-latest strategy: matrix: From 2dbb09d55c427cb3a4c6025ffa92841ed3b9a265 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 10:52:53 -0600 Subject: [PATCH 06/21] Add caching --- .github/workflows/run_tests.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 83a6e26106..0513fe0477 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -24,6 +24,7 @@ jobs: - uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: "Automatic Style fixes" + test: name: Run Tests needs: lint @@ -37,12 +38,22 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Cache Poetry virtual environment + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- - name: Install Poetry uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} - name: Install dependencies run: poetry install - name: Run tests with pytest run: poetry run pytest tests/ + build_poetry: name: Build Poetry needs: test @@ -56,6 +67,12 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- - name: Install Poetry uses: snok/install-poetry@v1 - name: Build @@ -66,6 +83,7 @@ jobs: run: python -c "import dspy" - name: Test import dsp run: python -c "import dsp" + build_setup: name: Build Setup needs: test @@ -79,5 +97,13 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + - name: Install Poetry + uses: snok/install-poetry@v1 - name: Run setup.py build run: python setup.py build From 471341b583c4766bef381d265a4e52cb53df9f09 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 11:03:43 -0600 Subject: [PATCH 07/21] Add import related rules --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c1ad1e8210..a1423285d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -218,6 +218,8 @@ select = [ "ERA", # pandas-vet "PD", + # Import sort + "I", ] ignore = [ "D100", From ab7ebdb7444160bbf1e4f2474be61141709e49e0 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 3 Mar 2024 17:04:03 +0000 Subject: [PATCH 08/21] Automatic Style fixes --- dsp/__init__.py | 1 - dsp/evaluation/utils.py | 5 ++-- dsp/modules/__init__.py | 26 +++++++++------------ dsp/modules/aws_lm.py | 5 ++-- dsp/modules/azurecognitivesearch.py | 3 ++- dsp/modules/bedrock.py | 3 ++- dsp/modules/cache_utils.py | 5 ++-- dsp/modules/clarifai.py | 1 + dsp/modules/cohere.py | 1 + dsp/modules/colbertv2.py | 4 ++-- dsp/modules/databricks.py | 2 +- dsp/modules/finetuning/finetune_hf.py | 20 ++++++++-------- dsp/modules/google.py | 3 ++- dsp/modules/gpt3.py | 4 ++-- dsp/modules/hf.py | 5 ++-- dsp/modules/hf_client.py | 13 +++++------ dsp/modules/ollama.py | 6 ++--- dsp/modules/pyserini.py | 5 ++-- dsp/primitives/__init__.py | 8 +++---- dsp/primitives/compiler.py | 7 +++--- dsp/primitives/demonstrate.py | 2 +- dsp/primitives/inspect.py | 5 ++-- dsp/primitives/predict.py | 8 +++---- dsp/primitives/primitives.py | 4 +++- dsp/templates/__init__.py | 2 +- dsp/templates/template_v2.py | 8 ++++--- dsp/templates/template_v3.py | 3 ++- dsp/templates/utils.py | 2 +- dsp/utils/__init__.py | 4 ++-- dsp/utils/dpr.py | 3 ++- dsp/utils/metrics.py | 2 +- dsp/utils/settings.py | 3 ++- dsp/utils/settings_v2.py | 5 ++-- dsp/utils/utils.py | 7 +++--- dspy/__init__.py | 18 +++++---------- dspy/datasets/__init__.py | 4 ++-- dspy/datasets/colors.py | 1 + dspy/datasets/dataloader.py | 10 ++++---- dspy/datasets/dataset.py | 5 ++-- dspy/datasets/gsm8k.py | 3 ++- dspy/datasets/hotpotqa.py | 1 + dspy/evaluate/__init__.py | 5 ++-- dspy/evaluate/auto_evaluation.py | 3 ++- dspy/evaluate/evaluate.py | 11 +++++---- dspy/evaluate/metrics.py | 1 + dspy/experimental/__init__.py | 2 +- dspy/experimental/synthesizer.py | 7 ++++-- dspy/experimental/synthetic_data.py | 7 ++++-- dspy/functional/__init__.py | 2 +- dspy/functional/functional.py | 14 +++++------ dspy/predict/__init__.py | 10 ++++---- dspy/predict/aggregation.py | 3 +-- dspy/predict/chain_of_thought.py | 1 - dspy/predict/chain_of_thought_with_hint.py | 2 -- dspy/predict/knn.py | 3 +++ dspy/predict/langchain.py | 7 +++--- dspy/predict/multi_chain_comparison.py | 4 ++-- dspy/predict/predict.py | 4 ++-- dspy/predict/program_of_thought.py | 4 +++- dspy/predict/react.py | 1 + dspy/predict/retry.py | 3 ++- dspy/primitives/__init__.py | 4 ++-- dspy/primitives/assertions.py | 5 ++-- dspy/primitives/module.py | 1 + dspy/primitives/program.py | 5 ++-- dspy/primitives/python_interpreter.py | 4 ++-- dspy/retrieve/chromadb_rm.py | 12 ++++++---- dspy/retrieve/databricks_rm.py | 9 +++++--- dspy/retrieve/deeplake_rm.py | 6 +++-- dspy/retrieve/marqo_rm.py | 1 + dspy/retrieve/mongodb_atlas_rm.py | 14 ++++++----- dspy/retrieve/pgvector_rm.py | 8 ++++--- dspy/retrieve/pinecone_rm.py | 9 +++++--- dspy/retrieve/qdrant_rm.py | 5 ++-- dspy/retrieve/retrieve.py | 5 ++-- dspy/retrieve/vectara_rm.py | 8 +++---- dspy/retrieve/weaviate_rm.py | 4 ++-- dspy/retrieve/you_rm.py | 7 +++--- dspy/signatures/signature.py | 9 ++++---- dspy/teleprompt/__init__.py | 10 ++++---- dspy/teleprompt/bootstrap.py | 7 +++--- dspy/teleprompt/finetune.py | 9 ++++---- dspy/teleprompt/knn_fewshot.py | 6 +++-- dspy/teleprompt/random_search.py | 4 +--- dspy/teleprompt/signature_opt.py | 7 +++--- dspy/teleprompt/signature_opt_bayesian.py | 16 +++++++------ dspy/teleprompt/teleprompt_optuna.py | 2 +- dspy/utils/dummies.py | 8 ++++--- examples/longformqa/utils.py | 4 +++- inspect-app/app.py | 7 +++--- setup.py | 2 +- testing/optimizer_tester.py | 25 +++++++++++--------- testing/tasks/__init__.py | 4 ++-- testing/tasks/base_task.py | 1 + testing/tasks/biodex.py | 27 +++++++++++++--------- testing/tasks/gsm8k.py | 11 +++++---- testing/tasks/hotpotqa.py | 4 +++- testing/tasks/scone.py | 6 ++++- testing/tasks/tweet.py | 12 ++++++---- testing/tasks/tweet_metric.py | 16 ++++++++----- 100 files changed, 354 insertions(+), 271 deletions(-) diff --git a/dsp/__init__.py b/dsp/__init__.py index fc8a742415..4f98315f13 100644 --- a/dsp/__init__.py +++ b/dsp/__init__.py @@ -3,7 +3,6 @@ from .templates import * from .utils import settings - """ TODO: diff --git a/dsp/evaluation/utils.py b/dsp/evaluation/utils.py index 1bfd740e4f..7b6a477802 100644 --- a/dsp/evaluation/utils.py +++ b/dsp/evaluation/utils.py @@ -1,7 +1,8 @@ -import dsp -import tqdm import pandas as pd +import tqdm + +import dsp try: from IPython.display import display as ipython_display diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py index d824d3fa57..241a349c58 100644 --- a/dsp/modules/__init__.py +++ b/dsp/modules/__init__.py @@ -1,19 +1,15 @@ -from .cache_utils import * from .azure_openai import AzureOpenAI -from .gpt3 import * +from .bedrock import * +from .cache_utils import * +from .clarifai import * +from .cohere import * +from .colbertv2 import ColBERTv2 from .databricks import * +from .google import * +from .gpt3 import * from .hf import HFModel -from .colbertv2 import ColBERTv2 -from .sentence_vectorizer import * -from .cohere import * -from .sbert import * -from .pyserini import * +from .hf_client import Anyscale, HFClientTGI, Together from .ollama import * -from .clarifai import * -from .bedrock import * -from .google import * - - -from .hf_client import HFClientTGI -from .hf_client import Anyscale -from .hf_client import Together +from .pyserini import * +from .sbert import * +from .sentence_vectorizer import * diff --git a/dsp/modules/aws_lm.py b/dsp/modules/aws_lm.py index 83674db10a..a44e30a492 100644 --- a/dsp/modules/aws_lm.py +++ b/dsp/modules/aws_lm.py @@ -4,10 +4,11 @@ from __future__ import annotations -from abc import abstractmethod +import json import logging +from abc import abstractmethod from typing import Any, Literal -import json + from dsp.modules.lm import LM # Heuristic translating number of chars to tokens diff --git a/dsp/modules/azurecognitivesearch.py b/dsp/modules/azurecognitivesearch.py index e4ff2e9960..fedc59b015 100644 --- a/dsp/modules/azurecognitivesearch.py +++ b/dsp/modules/azurecognitivesearch.py @@ -1,6 +1,7 @@ -from typing import Union, Any +from typing import Any, Union from dsp.utils import dotdict + try: from azure.core.credentials import AzureKeyCredential from azure.search.documents import SearchClient diff --git a/dsp/modules/bedrock.py b/dsp/modules/bedrock.py index c52ae61970..ac8b42d2d4 100644 --- a/dsp/modules/bedrock.py +++ b/dsp/modules/bedrock.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import Any import json +from typing import Any + from dsp.modules.aws_lm import AWSLM diff --git a/dsp/modules/cache_utils.py b/dsp/modules/cache_utils.py index 78270c879a..e6c09fe716 100644 --- a/dsp/modules/cache_utils.py +++ b/dsp/modules/cache_utils.py @@ -1,12 +1,11 @@ import os - +from functools import wraps from pathlib import Path + from joblib import Memory -from functools import wraps from dsp.utils import dotdict - cache_turn_on = True diff --git a/dsp/modules/clarifai.py b/dsp/modules/clarifai.py index 8a691a4bef..2d5839c62b 100644 --- a/dsp/modules/clarifai.py +++ b/dsp/modules/clarifai.py @@ -3,6 +3,7 @@ from dsp.modules.lm import LM + class ClarifaiLLM(LM): """Integration to call models hosted in clarifai platform. diff --git a/dsp/modules/cohere.py b/dsp/modules/cohere.py index 405eaea483..7308d59355 100644 --- a/dsp/modules/cohere.py +++ b/dsp/modules/cohere.py @@ -1,5 +1,6 @@ import math from typing import Any, Optional + import backoff from dsp.modules.lm import LM diff --git a/dsp/modules/colbertv2.py b/dsp/modules/colbertv2.py index 1b33d077a9..8ff3c16225 100644 --- a/dsp/modules/colbertv2.py +++ b/dsp/modules/colbertv2.py @@ -1,11 +1,11 @@ import functools -from typing import Optional, Union, Any +from typing import Any, Optional, Union + import requests from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory from dsp.utils import dotdict - # TODO: Ideally, this takes the name of the index and looks up its port. diff --git a/dsp/modules/databricks.py b/dsp/modules/databricks.py index 8f7d7d6e87..7d90e4fc27 100644 --- a/dsp/modules/databricks.py +++ b/dsp/modules/databricks.py @@ -19,8 +19,8 @@ from dsp.modules.gpt3 import GPT3 try: - from openai.openai_object import OpenAIObject import openai.error + from openai.openai_object import OpenAIObject ERRORS = (openai.error.RateLimitError, openai.error.ServiceUnavailableError, openai.error.APIError) except Exception: ERRORS = (openai.RateLimitError, openai.APIError) diff --git a/dsp/modules/finetuning/finetune_hf.py b/dsp/modules/finetuning/finetune_hf.py index 0ade213bae..a22ab10fcb 100644 --- a/dsp/modules/finetuning/finetune_hf.py +++ b/dsp/modules/finetuning/finetune_hf.py @@ -1,28 +1,30 @@ # Adapted from: https://www.philschmid.de/fine-tune-flan-t5#3-fine-tune-and-evaluate-flan-t5 -import os -import json import copy import glob -import torch +import json +import os import warnings +from dataclasses import dataclass + import evaluate import numpy as np +import torch from datasets import Dataset -from dataclasses import dataclass from transformers import ( - set_seed, AutoConfig, - AutoModelForSeq2SeqLM, AutoModelForCausalLM, + AutoModelForSeq2SeqLM, AutoTokenizer, + DataCollatorForSeq2Seq, PreTrainedTokenizer, - Trainer, Seq2SeqTrainer, - TrainingArguments, Seq2SeqTrainingArguments, - DataCollatorForSeq2Seq, + Trainer, + TrainingArguments, + set_seed, ) + # from peft import get_peft_model, LoraConfig, TaskType from transformers.trainer_callback import TrainerCallback diff --git a/dsp/modules/google.py b/dsp/modules/google.py index 9a1c3937c1..5d97534ea8 100644 --- a/dsp/modules/google.py +++ b/dsp/modules/google.py @@ -1,6 +1,7 @@ import os -from typing import Any, Optional from collections.abc import Iterable +from typing import Any, Optional + import backoff from dsp.modules.lm import LM diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.py index ad440f2348..d0b343af3c 100644 --- a/dsp/modules/gpt3.py +++ b/dsp/modules/gpt3.py @@ -11,10 +11,10 @@ import json from typing import Any, Literal, Optional, cast -import dsp import backoff import openai +import dsp from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on from dsp.modules.lm import LM @@ -24,8 +24,8 @@ OPENAI_LEGACY = True try: - from openai.openai_object import OpenAIObject import openai.error + from openai.openai_object import OpenAIObject ERRORS = ( openai.error.RateLimitError, diff --git a/dsp/modules/hf.py b/dsp/modules/hf.py index 3c9306a8f8..aad0c0e36c 100644 --- a/dsp/modules/hf.py +++ b/dsp/modules/hf.py @@ -1,8 +1,9 @@ # from peft import PeftConfig, PeftModel # from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer, AutoConfig -from typing import Optional, Literal +from typing import Literal, Optional from dsp.modules.lm import LM + # from dsp.modules.finetuning.finetune_hf import preprocess_prompt def openai_to_hf(**kwargs): @@ -43,8 +44,8 @@ def __init__(self, model: str, checkpoint: Optional[str] = None, is_client: bool self.device_map = hf_device_map if not self.is_client: try: - from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer, AutoConfig import torch + from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer except ImportError as exc: raise ModuleNotFoundError( "You need to install Hugging Face transformers library to use HF models.", diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py index 4def628199..428b0d06e6 100644 --- a/dsp/modules/hf_client.py +++ b/dsp/modules/hf_client.py @@ -1,15 +1,15 @@ import os import random -import requests -from dsp.modules.hf import HFModel, openai_to_hf -from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory -import subprocess import re import shutil +import subprocess # from dsp.modules.adapter import TurboAdapter, DavinciAdapter, LlamaAdapter - import backoff +import requests + +from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory +from dsp.modules.hf import HFModel, openai_to_hf ERRORS = (Exception) @@ -355,8 +355,7 @@ class ChatModuleClient(HFModel): def __init__(self, model, model_path): super().__init__(model=model, is_client=True) - from mlc_chat import ChatModule - from mlc_chat import ChatConfig + from mlc_chat import ChatConfig, ChatModule self.cm = ChatModule( model=model, lib_path=model_path, chat_config=ChatConfig(conv_template="LM"), diff --git a/dsp/modules/ollama.py b/dsp/modules/ollama.py index a2e3e01397..27304d271e 100644 --- a/dsp/modules/ollama.py +++ b/dsp/modules/ollama.py @@ -1,10 +1,10 @@ -from dsp.modules.lm import LM -from typing import Any, Literal - import datetime import hashlib +from typing import Any, Literal + import requests +from dsp.modules.lm import LM def post_request_metadata(model_name, prompt): diff --git a/dsp/modules/pyserini.py b/dsp/modules/pyserini.py index 94ce970ea2..5523a76e77 100644 --- a/dsp/modules/pyserini.py +++ b/dsp/modules/pyserini.py @@ -1,5 +1,6 @@ -from typing import Union import json +from typing import Union + from datasets import Dataset from dsp.utils import dotdict @@ -30,8 +31,8 @@ def __init__(self, """ # Keep pyserini as an optional dependency + from pyserini.prebuilt_index_info import FAISS_INDEX_INFO, IMPACT_INDEX_INFO, TF_INDEX_INFO from pyserini.search import FaissSearcher - from pyserini.prebuilt_index_info import TF_INDEX_INFO, FAISS_INDEX_INFO, IMPACT_INDEX_INFO self.encoder = FaissSearcher._init_encoder_from_str(query_encoder) self.dataset = dataset diff --git a/dsp/primitives/__init__.py b/dsp/primitives/__init__.py index 3798444a1e..ef278efa6b 100644 --- a/dsp/primitives/__init__.py +++ b/dsp/primitives/__init__.py @@ -1,6 +1,6 @@ -from .primitives import * +from .compiler import * from .demonstrate import * -from .search import * -from .predict import * from .inspect import * -from .compiler import * +from .predict import * +from .primitives import * +from .search import * diff --git a/dsp/primitives/compiler.py b/dsp/primitives/compiler.py index 996919e856..e5bcd98b11 100644 --- a/dsp/primitives/compiler.py +++ b/dsp/primitives/compiler.py @@ -1,12 +1,13 @@ import os +import random +import subprocess import time + import tqdm import ujson -import random -import subprocess +from datasets.fingerprint import Hasher import dsp -from datasets.fingerprint import Hasher if os.environ.get('DSP_NOTEBOOK_CACHEDIR'): training_data_directory = os.path.join(os.environ.get('DSP_NOTEBOOK_CACHEDIR'), 'compiler') diff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.py index ae08e28bae..7dfe126b53 100644 --- a/dsp/primitives/demonstrate.py +++ b/dsp/primitives/demonstrate.py @@ -1,5 +1,5 @@ import random -from typing import Callable, Any +from typing import Any, Callable import numpy as np diff --git a/dsp/primitives/inspect.py b/dsp/primitives/inspect.py index 3fbb05c55a..ebcd3b7872 100644 --- a/dsp/primitives/inspect.py +++ b/dsp/primitives/inspect.py @@ -1,8 +1,9 @@ import inspect -import string +import json import random +import string + import requests -import json class FuncInspector: diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.py index c89256b916..41edf6c51c 100644 --- a/dsp/primitives/predict.py +++ b/dsp/primitives/predict.py @@ -1,11 +1,11 @@ from collections import Counter -from typing import Callable, Any, Optional +from typing import Any, Callable, Optional import dsp -from dsp.utils import zipstar, normalize_text -from dsp.utils.utils import dotdict -from dsp.templates.template_v3 import Template from dsp.primitives.demonstrate import Example +from dsp.templates.template_v3 import Template +from dsp.utils import normalize_text, zipstar +from dsp.utils.utils import dotdict class Completions: diff --git a/dsp/primitives/primitives.py b/dsp/primitives/primitives.py index a7851ea33d..2e4c4bfc07 100644 --- a/dsp/primitives/primitives.py +++ b/dsp/primitives/primitives.py @@ -1,6 +1,8 @@ -import dsp from functools import wraps +import dsp + + # applied right to left (innermost first, like function calls) def compose_decorators(*decorators): def decorator(func): diff --git a/dsp/templates/__init__.py b/dsp/templates/__init__.py index b4aed76438..5eaae95f75 100644 --- a/dsp/templates/__init__.py +++ b/dsp/templates/__init__.py @@ -1,4 +1,4 @@ -from .utils import * from .template_v2 import * from .template_v3 import * +from .utils import * diff --git a/dsp/templates/template_v2.py b/dsp/templates/template_v2.py index d6e61642ea..236fd1d64e 100644 --- a/dsp/templates/template_v2.py +++ b/dsp/templates/template_v2.py @@ -1,9 +1,11 @@ -from collections import namedtuple import re -from typing import Union, Any +from collections import namedtuple +from typing import Any, Union + import dsp from dsp.primitives.demonstrate import Example -from .utils import passages2text, format_answers + +from .utils import format_answers, passages2text Field = namedtuple("Field", "name separator input_variable output_variable description") diff --git a/dsp/templates/template_v3.py b/dsp/templates/template_v3.py index fb097f8770..605705c153 100644 --- a/dsp/templates/template_v3.py +++ b/dsp/templates/template_v3.py @@ -1,5 +1,6 @@ from typing import Callable -from dsp.templates import TemplateV2, passages2text, format_answers, Field + +from dsp.templates import Field, TemplateV2, format_answers, passages2text class Type: diff --git a/dsp/templates/utils.py b/dsp/templates/utils.py index 0b0d4891f3..53624ad2d8 100644 --- a/dsp/templates/utils.py +++ b/dsp/templates/utils.py @@ -1,4 +1,4 @@ -from typing import Union, Optional +from typing import Optional, Union def passages2text(passages: Union[str, list, tuple]) -> str: diff --git a/dsp/utils/__init__.py b/dsp/utils/__init__.py index c41f748328..63624350a7 100644 --- a/dsp/utils/__init__.py +++ b/dsp/utils/__init__.py @@ -1,4 +1,4 @@ +from dsp.utils.dpr import * +from dsp.utils.metrics import * from dsp.utils.settings import * from dsp.utils.utils import * -from dsp.utils.metrics import * -from dsp.utils.dpr import * diff --git a/dsp/utils/dpr.py b/dsp/utils/dpr.py index 2174ea4da6..d4d18f84ff 100644 --- a/dsp/utils/dpr.py +++ b/dsp/utils/dpr.py @@ -4,9 +4,10 @@ Original license: https://github.com/facebookresearch/DPR/blob/main/LICENSE """ -import regex import unicodedata +import regex + class Tokens: """A class to represent a list of tokenized text.""" diff --git a/dsp/utils/metrics.py b/dsp/utils/metrics.py index f0d69f00f6..ddbd51ae94 100644 --- a/dsp/utils/metrics.py +++ b/dsp/utils/metrics.py @@ -1,8 +1,8 @@ import re import string import unicodedata - from collections import Counter + from dsp.utils.utils import print_message diff --git a/dsp/utils/settings.py b/dsp/utils/settings.py index 462f99d429..1de0ada7ec 100644 --- a/dsp/utils/settings.py +++ b/dsp/utils/settings.py @@ -1,6 +1,7 @@ +import threading from contextlib import contextmanager + from dsp.utils.utils import dotdict -import threading class Settings: diff --git a/dsp/utils/settings_v2.py b/dsp/utils/settings_v2.py index 2998ca7cd4..6652474d3d 100644 --- a/dsp/utils/settings_v2.py +++ b/dsp/utils/settings_v2.py @@ -1,7 +1,8 @@ +import copy import threading -from contextlib import contextmanager from concurrent.futures import ThreadPoolExecutor, as_completed -import copy +from contextlib import contextmanager + class Settings: def __init__(self): diff --git a/dsp/utils/utils.py b/dsp/utils/utils.py index aeba5c9700..d5a30b0e39 100644 --- a/dsp/utils/utils.py +++ b/dsp/utils/utils.py @@ -1,10 +1,10 @@ -import os -import tqdm import datetime import itertools - +import os from collections import defaultdict +import tqdm + def print_message(*s, condition=True, pad=False, sep=None): s = " ".join([str(x) for x in s]) @@ -77,6 +77,7 @@ def batch(group, bsize, provide_offset=False): import copy + class dotdict(dict): def __getattr__(self, key): if key.startswith('__') and key.endswith('__'): diff --git a/dspy/__init__.py b/dspy/__init__.py index 98fe6fba89..b61ea8a017 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -1,18 +1,12 @@ -from dsp.modules.hf_client import ChatModuleClient -from dsp.modules.hf_client import HFServerTGI, HFClientVLLM, HFClientSGLang -from .signatures import * - -from .retrieve import * -from .predict import * -from .primitives import * - # from .evaluation import * - - # FIXME: - - import dsp +from dsp.modules.hf_client import ChatModuleClient, HFClientSGLang, HFClientVLLM, HFServerTGI + +from .predict import * +from .primitives import * +from .retrieve import * +from .signatures import * settings = dsp.settings diff --git a/dspy/datasets/__init__.py b/dspy/datasets/__init__.py index 62409d6ed4..bf6e3dd019 100644 --- a/dspy/datasets/__init__.py +++ b/dspy/datasets/__init__.py @@ -1,4 +1,4 @@ +from .colors import Colors +from .dataloader import DataLoader from .dataset import Dataset from .hotpotqa import HotPotQA -from .colors import Colors -from .dataloader import DataLoader \ No newline at end of file diff --git a/dspy/datasets/colors.py b/dspy/datasets/colors.py index 265badf270..edf2e41697 100644 --- a/dspy/datasets/colors.py +++ b/dspy/datasets/colors.py @@ -1,4 +1,5 @@ import random + from dspy.datasets.dataset import Dataset ### A bunch of colors, originally from matplotlib diff --git a/dspy/datasets/dataloader.py b/dspy/datasets/dataloader.py index 8900c107f5..f4d3b87580 100644 --- a/dspy/datasets/dataloader.py +++ b/dspy/datasets/dataloader.py @@ -1,10 +1,12 @@ -import dspy import random -from dspy.datasets import Dataset +from collections.abc import Mapping +from typing import List, Tuple, Union from datasets import load_dataset -from typing import Union, List, Tuple -from collections.abc import Mapping + +import dspy +from dspy.datasets import Dataset + class DataLoader(Dataset): def __init__(self,): diff --git a/dspy/datasets/dataset.py b/dspy/datasets/dataset.py index 9cd279199f..c66c5edc23 100644 --- a/dspy/datasets/dataset.py +++ b/dspy/datasets/dataset.py @@ -1,8 +1,9 @@ -import uuid import random +import uuid -from dspy import Example from dsp.utils import dotdict +from dspy import Example + class Dataset: def __init__(self, train_seed=0, train_size=None, eval_seed=0, dev_size=None, test_size=None): diff --git a/dspy/datasets/gsm8k.py b/dspy/datasets/gsm8k.py index 3b3514862d..328d1aee8d 100644 --- a/dspy/datasets/gsm8k.py +++ b/dspy/datasets/gsm8k.py @@ -1,8 +1,9 @@ -import tqdm import random +import tqdm from datasets import load_dataset + class GSM8K: def __init__(self) -> None: super().__init__() diff --git a/dspy/datasets/hotpotqa.py b/dspy/datasets/hotpotqa.py index f1cc734824..9fbf086179 100644 --- a/dspy/datasets/hotpotqa.py +++ b/dspy/datasets/hotpotqa.py @@ -1,6 +1,7 @@ import random from datasets import load_dataset + from dspy.datasets.dataset import Dataset diff --git a/dspy/evaluate/__init__.py b/dspy/evaluate/__init__.py index eea386c599..2526c2be07 100644 --- a/dspy/evaluate/__init__.py +++ b/dspy/evaluate/__init__.py @@ -1,4 +1,5 @@ +from dsp.utils import EM, normalize_text + +from .auto_evaluation import * from .evaluate import Evaluate from .metrics import * -from .auto_evaluation import * -from dsp.utils import EM, normalize_text diff --git a/dspy/evaluate/auto_evaluation.py b/dspy/evaluate/auto_evaluation.py index 1ec659540c..569a5b16c8 100644 --- a/dspy/evaluate/auto_evaluation.py +++ b/dspy/evaluate/auto_evaluation.py @@ -1,4 +1,5 @@ -import dspy +import dspy + class AnswerCorrectnessSignature(dspy.Signature): """Verify that the predicted answer matches the gold answer.""" diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.py index 0be330375b..f38bcff38f 100644 --- a/dspy/evaluate/evaluate.py +++ b/dspy/evaluate/evaluate.py @@ -1,11 +1,14 @@ -import dsp -import tqdm -import types import threading +import types + import pandas as pd +import tqdm + +import dsp try: - from IPython.display import display as ipython_display, HTML + from IPython.display import HTML + from IPython.display import display as ipython_display except ImportError: ipython_display = print HTML = lambda x: x diff --git a/dspy/evaluate/metrics.py b/dspy/evaluate/metrics.py index 79c6208af6..0e25b5a119 100644 --- a/dspy/evaluate/metrics.py +++ b/dspy/evaluate/metrics.py @@ -2,6 +2,7 @@ import dsp + def answer_exact_match(example, pred, trace=None, frac=1.0): assert(type(example.answer) is str or type(example.answer) is list) diff --git a/dspy/experimental/__init__.py b/dspy/experimental/__init__.py index b4385639db..36477304c3 100644 --- a/dspy/experimental/__init__.py +++ b/dspy/experimental/__init__.py @@ -1,2 +1,2 @@ -from .synthetic_data import * from .synthesizer import * +from .synthetic_data import * diff --git a/dspy/experimental/synthesizer.py b/dspy/experimental/synthesizer.py index d766f2dcab..a66bf4d2ba 100644 --- a/dspy/experimental/synthesizer.py +++ b/dspy/experimental/synthesizer.py @@ -1,8 +1,11 @@ -import dspy import random from typing import List -from tqdm import tqdm, trange + from datasets import Dataset +from tqdm import tqdm, trange + +import dspy + def format_examples(examples: List[dspy.Example]): if isinstance(examples, str): diff --git a/dspy/experimental/synthetic_data.py b/dspy/experimental/synthetic_data.py index b9121b98d1..37c1403522 100644 --- a/dspy/experimental/synthetic_data.py +++ b/dspy/experimental/synthetic_data.py @@ -1,8 +1,11 @@ -from pydantic import BaseModel -import dspy import random from typing import List, Optional +from pydantic import BaseModel + +import dspy + + class descriptionSignature(dspy.Signature): field_name = dspy.InputField(desc="name of a field") example = dspy.InputField(desc="an example value for the field") diff --git a/dspy/functional/__init__.py b/dspy/functional/__init__.py index 3ccced47b9..746ac93181 100644 --- a/dspy/functional/__init__.py +++ b/dspy/functional/__init__.py @@ -1 +1 @@ -from .functional import cot, predictor, FunctionalModule, TypedPredictor, TypedChainOfThought +from .functional import FunctionalModule, TypedChainOfThought, TypedPredictor, cot, predictor diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index 96c218ebbd..315727a374 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -1,15 +1,15 @@ import inspect +import json import os -import openai -import dspy import typing -import pydantic from typing import Annotated, List, Tuple # noqa: UP035 -from dsp.templates import passages2text -import json -from dspy.signatures.signature import ensure_signature +import openai +import pydantic +import dspy +from dsp.templates import passages2text +from dspy.signatures.signature import ensure_signature MAX_RETRIES = 3 @@ -314,8 +314,8 @@ def gold_passages_retrieved(example, pred, _trace=None) -> bool: def hotpot() -> None: - from dsp.utils import deduplicate import dspy.evaluate + from dsp.utils import deduplicate from dspy.datasets import HotPotQA from dspy.evaluate.evaluate import Evaluate from dspy.teleprompt.bootstrap import BootstrapFewShot diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py index 8b0770150b..b176d2a0ed 100644 --- a/dspy/predict/__init__.py +++ b/dspy/predict/__init__.py @@ -1,9 +1,9 @@ -from .predict import Predict +from .aggregation import majority from .chain_of_thought import ChainOfThought -from .multi_chain_comparison import MultiChainComparison from .chain_of_thought_with_hint import ChainOfThoughtWithHint -from .react import ReAct -from .aggregation import majority +from .knn import KNN +from .multi_chain_comparison import MultiChainComparison +from .predict import Predict from .program_of_thought import ProgramOfThought +from .react import ReAct from .retry import Retry -from .knn import KNN \ No newline at end of file diff --git a/dspy/predict/aggregation.py b/dspy/predict/aggregation.py index ca4154aa2d..4cb6df3f91 100644 --- a/dspy/predict/aggregation.py +++ b/dspy/predict/aggregation.py @@ -1,6 +1,5 @@ -from dspy.primitives.prediction import Prediction, Completions from dsp.utils import normalize_text - +from dspy.primitives.prediction import Completions, Prediction default_normalize = lambda s: normalize_text(s) or None diff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.py index f374aaabe5..70bab0a7ae 100644 --- a/dspy/predict/chain_of_thought.py +++ b/dspy/predict/chain_of_thought.py @@ -4,7 +4,6 @@ from .predict import Predict - # TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system. # TODO: This shouldn't inherit from Predict. It should be a module that has one or two predictors. diff --git a/dspy/predict/chain_of_thought_with_hint.py b/dspy/predict/chain_of_thought_with_hint.py index 6f78566321..484a2a38a0 100644 --- a/dspy/predict/chain_of_thought_with_hint.py +++ b/dspy/predict/chain_of_thought_with_hint.py @@ -3,8 +3,6 @@ from .predict import Predict - - # TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system. class ChainOfThoughtWithHint(Predict): diff --git a/dspy/predict/knn.py b/dspy/predict/knn.py index e0e49fe2b4..e275ccb6ac 100644 --- a/dspy/predict/knn.py +++ b/dspy/predict/knn.py @@ -1,7 +1,10 @@ from typing import List + import numpy as np + import dsp + class KNN: def __init__(self, k: int, trainset: List[dsp.Example]): self.k = k diff --git a/dspy/predict/langchain.py b/dspy/predict/langchain.py index 4be855e8db..86d439f50c 100644 --- a/dspy/predict/langchain.py +++ b/dspy/predict/langchain.py @@ -1,18 +1,17 @@ import copy import random +from langchain_core.pydantic_v1 import Extra +from langchain_core.runnables import Runnable + import dsp import dspy - from dspy.predict.parameter import Parameter from dspy.predict.predict import Predict from dspy.primitives.prediction import Prediction from dspy.signatures.field import InputField, OutputField from dspy.signatures.signature import infer_prefix -from langchain_core.pydantic_v1 import Extra -from langchain_core.runnables import Runnable - # TODO: This class is currently hard to test, because it hardcodes gpt-4 usage: # gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=4000, model_type='chat') diff --git a/dspy/predict/multi_chain_comparison.py b/dspy/predict/multi_chain_comparison.py index ad69b646ee..37b66afadd 100644 --- a/dspy/predict/multi_chain_comparison.py +++ b/dspy/predict/multi_chain_comparison.py @@ -1,8 +1,8 @@ import dspy from dspy.signatures.signature import ensure_signature -from .predict import Predict -from ..primitives.program import Module +from ..primitives.program import Module +from .predict import Predict class MultiChainComparison(Module): diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py index 60e668a836..fe6beb4483 100644 --- a/dspy/predict/predict.py +++ b/dspy/predict/predict.py @@ -1,11 +1,11 @@ -import dsp import random +import dsp from dspy.predict.parameter import Parameter from dspy.primitives.prediction import Prediction - from dspy.signatures.signature import ensure_signature, signature_to_template + class Predict(Parameter): def __init__(self, signature, **config): self.stage = random.randbytes(8).hex() diff --git a/dspy/predict/program_of_thought.py b/dspy/predict/program_of_thought.py index f7e80eed71..1cfb54bd99 100644 --- a/dspy/predict/program_of_thought.py +++ b/dspy/predict/program_of_thought.py @@ -1,8 +1,10 @@ +import re + import dspy from dspy.signatures.signature import ensure_signature + from ..primitives.program import Module from ..primitives.python_interpreter import CodePrompt, PythonInterpreter -import re class ProgramOfThought(Module): diff --git a/dspy/predict/react.py b/dspy/predict/react.py index cf3be8b73c..a465ee710b 100644 --- a/dspy/predict/react.py +++ b/dspy/predict/react.py @@ -1,6 +1,7 @@ import dsp import dspy from dspy.signatures.signature import ensure_signature + from ..primitives.program import Module from .predict import Predict diff --git a/dspy/predict/retry.py b/dspy/predict/retry.py index af1d37f98b..dcef1488f6 100644 --- a/dspy/predict/retry.py +++ b/dspy/predict/retry.py @@ -1,6 +1,7 @@ import copy -import dspy + import dsp +import dspy from .predict import Predict diff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.py index b7baf229bf..e667837166 100644 --- a/dspy/primitives/__init__.py +++ b/dspy/primitives/__init__.py @@ -1,5 +1,5 @@ +from .assertions import * from .example import * -from .program import * from .prediction import * -from .assertions import * +from .program import * from .python_interpreter import * diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py index 140f8bf6a2..19f1aea3a1 100644 --- a/dspy/primitives/assertions.py +++ b/dspy/primitives/assertions.py @@ -1,9 +1,10 @@ import inspect +import logging +import uuid from typing import Any + import dsp import dspy -import logging -import uuid #################### Assertion Helpers #################### diff --git a/dspy/primitives/module.py b/dspy/primitives/module.py index 1b4a342f71..bee80338f5 100644 --- a/dspy/primitives/module.py +++ b/dspy/primitives/module.py @@ -1,4 +1,5 @@ import copy + import ujson diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py index 5e063bfb00..aa499d9085 100644 --- a/dspy/primitives/program.py +++ b/dspy/primitives/program.py @@ -1,8 +1,9 @@ -from dspy.primitives.module import BaseModule -from dspy.primitives.assertions import * import re +from dspy.primitives.assertions import * +from dspy.primitives.module import BaseModule + class ProgramMeta(type): pass diff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.py index fd6166db5e..1b7456c7b0 100644 --- a/dspy/primitives/python_interpreter.py +++ b/dspy/primitives/python_interpreter.py @@ -12,10 +12,12 @@ # limitations under the License. # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. =========== import ast +import builtins import difflib import importlib import re import typing +from collections.abc import Mapping from typing import ( Any, Dict, @@ -24,8 +26,6 @@ Set, Tuple, ) -from collections.abc import Mapping -import builtins class InterpreterError(ValueError): diff --git a/dspy/retrieve/chromadb_rm.py b/dspy/retrieve/chromadb_rm.py index b4fc0bcf98..07ef407d1d 100644 --- a/dspy/retrieve/chromadb_rm.py +++ b/dspy/retrieve/chromadb_rm.py @@ -2,10 +2,12 @@ Retriever model for chromadb """ -from typing import Optional, List, Union +from typing import List, Optional, Union + +import backoff import openai + import dspy -import backoff from dsp.utils import dotdict try: @@ -16,13 +18,13 @@ try: import chromadb - from chromadb.config import Settings - from chromadb.utils import embedding_functions + import chromadb.utils.embedding_functions as ef from chromadb.api.types import ( Embeddable, EmbeddingFunction, ) - import chromadb.utils.embedding_functions as ef + from chromadb.config import Settings + from chromadb.utils import embedding_functions except ImportError: chromadb = None diff --git a/dspy/retrieve/databricks_rm.py b/dspy/retrieve/databricks_rm.py index 74066ec5b9..c275bdddc2 100644 --- a/dspy/retrieve/databricks_rm.py +++ b/dspy/retrieve/databricks_rm.py @@ -1,10 +1,13 @@ -import dspy import os -import requests -from typing import Union, List from collections import defaultdict +from typing import List, Union + +import requests + +import dspy from dspy.primitives.prediction import Prediction + class DatabricksRM(dspy.Retrieve): """ A retrieval module that uses Databricks Vector Search Endpoint to return the top-k embeddings for a given query. diff --git a/dspy/retrieve/deeplake_rm.py b/dspy/retrieve/deeplake_rm.py index 87beb48565..235108a912 100644 --- a/dspy/retrieve/deeplake_rm.py +++ b/dspy/retrieve/deeplake_rm.py @@ -2,10 +2,12 @@ Retriever model for deeplake """ -from typing import Optional, List, Union +from collections import defaultdict +from typing import List, Optional, Union + import openai + import dspy -from collections import defaultdict from dsp.utils import dotdict try: diff --git a/dspy/retrieve/marqo_rm.py b/dspy/retrieve/marqo_rm.py index a090e47f52..29c52fdb46 100644 --- a/dspy/retrieve/marqo_rm.py +++ b/dspy/retrieve/marqo_rm.py @@ -1,5 +1,6 @@ from collections import defaultdict from typing import List, Union + import dspy from dspy import dotdict diff --git a/dspy/retrieve/mongodb_atlas_rm.py b/dspy/retrieve/mongodb_atlas_rm.py index 70e7847303..3cafb05f65 100644 --- a/dspy/retrieve/mongodb_atlas_rm.py +++ b/dspy/retrieve/mongodb_atlas_rm.py @@ -1,23 +1,25 @@ -from typing import List, Any -import dspy import os +from typing import Any, List + +import backoff from openai import ( - OpenAI, APITimeoutError, InternalServerError, + OpenAI, RateLimitError, UnprocessableEntityError, ) -import backoff + +import dspy try: from pymongo import MongoClient from pymongo.errors import ( - ConnectionFailure, ConfigurationError, - ServerSelectionTimeoutError, + ConnectionFailure, InvalidURI, OperationFailure, + ServerSelectionTimeoutError, ) except ImportError: raise ImportError( diff --git a/dspy/retrieve/pgvector_rm.py b/dspy/retrieve/pgvector_rm.py index 8403efb2e3..cf1773f171 100644 --- a/dspy/retrieve/pgvector_rm.py +++ b/dspy/retrieve/pgvector_rm.py @@ -1,10 +1,12 @@ -import dspy -import openai from typing import List, Optional +import openai + +import dspy + try: - from pgvector.psycopg2 import register_vector import psycopg2 + from pgvector.psycopg2 import register_vector from psycopg2 import sql except ImportError: raise ImportError( diff --git a/dspy/retrieve/pinecone_rm.py b/dspy/retrieve/pinecone_rm.py index b520ed5fac..0328bc6ba1 100644 --- a/dspy/retrieve/pinecone_rm.py +++ b/dspy/retrieve/pinecone_rm.py @@ -3,11 +3,13 @@ Author: Dhar Rawal (@drawal1) """ -from dsp.utils import dotdict -from typing import Optional, List, Union -import dspy +from typing import List, Optional, Union + import backoff +import dspy +from dsp.utils import dotdict + try: import pinecone except ImportError: @@ -19,6 +21,7 @@ ) import openai + try: OPENAI_LEGACY = int(openai.version.__version__[0]) == 0 except Exception: diff --git a/dspy/retrieve/qdrant_rm.py b/dspy/retrieve/qdrant_rm.py index 46977a2524..5c2af050b9 100644 --- a/dspy/retrieve/qdrant_rm.py +++ b/dspy/retrieve/qdrant_rm.py @@ -1,11 +1,12 @@ from collections import defaultdict -from typing import List, Union, Optional +from typing import List, Optional, Union + import dspy from dsp.utils import dotdict try: - from qdrant_client import QdrantClient import fastembed + from qdrant_client import QdrantClient except ImportError: raise ImportError( "The 'qdrant' extra is required to use QdrantRM. Install it with `pip install dspy-ai[qdrant]`", diff --git a/dspy/retrieve/retrieve.py b/dspy/retrieve/retrieve.py index 9108582414..71b4f57349 100644 --- a/dspy/retrieve/retrieve.py +++ b/dspy/retrieve/retrieve.py @@ -1,8 +1,7 @@ -from typing import List, Union, Optional - -import dsp import random +from typing import List, Optional, Union +import dsp from dspy.predict.parameter import Parameter from dspy.primitives.prediction import Prediction diff --git a/dspy/retrieve/vectara_rm.py b/dspy/retrieve/vectara_rm.py index 4d72ed7e87..047c70d6c9 100644 --- a/dspy/retrieve/vectara_rm.py +++ b/dspy/retrieve/vectara_rm.py @@ -1,11 +1,11 @@ -from collections import defaultdict -from typing import List, Union -import dspy -from typing import Optional import json import os +from collections import defaultdict +from typing import List, Optional, Union + import requests +import dspy from dsp.utils import dotdict START_SNIPPET = "<%START%>" diff --git a/dspy/retrieve/weaviate_rm.py b/dspy/retrieve/weaviate_rm.py index 1ef7950c1f..61e8d1ef06 100644 --- a/dspy/retrieve/weaviate_rm.py +++ b/dspy/retrieve/weaviate_rm.py @@ -1,7 +1,7 @@ -from typing import List, Union +from typing import List, Optional, Union + import dspy from dsp.utils import dotdict -from typing import Optional try: import weaviate diff --git a/dspy/retrieve/you_rm.py b/dspy/retrieve/you_rm.py index 7321e67bd3..25a62ccea8 100644 --- a/dspy/retrieve/you_rm.py +++ b/dspy/retrieve/you_rm.py @@ -1,9 +1,10 @@ -import dspy import os +from typing import List, Optional, Union + import requests -from dsp.utils import dotdict -from typing import Union, List, Optional +import dspy +from dsp.utils import dotdict class YouRM(dspy.Retrieve): diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py index 7cd2f170cb..7b552c9f03 100644 --- a/dspy/signatures/signature.py +++ b/dspy/signatures/signature.py @@ -1,11 +1,12 @@ -from copy import deepcopy +import re import typing -import dsp +from copy import deepcopy +from typing import Dict, Tuple, Type, Union + from pydantic import BaseModel, Field, create_model from pydantic.fields import FieldInfo -from typing import Type, Union, Dict, Tuple -import re +import dsp from dspy.signatures.field import InputField, OutputField, new_to_old_field diff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.py index c4454725bd..61425e5b9b 100644 --- a/dspy/teleprompt/__init__.py +++ b/dspy/teleprompt/__init__.py @@ -1,9 +1,9 @@ -from .teleprompt import * from .bootstrap import * -from .vanilla import * -from .random_search import * from .finetune import * -from .teleprompt_optuna import * from .knn_fewshot import * +from .random_search import * from .signature_opt import SignatureOptimizer -from .signature_opt_bayesian import BayesianSignatureOptimizer \ No newline at end of file +from .signature_opt_bayesian import BayesianSignatureOptimizer +from .teleprompt import * +from .teleprompt_optuna import * +from .vanilla import * diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py index 5aaefa271f..6fc9dd2ef7 100644 --- a/dspy/teleprompt/bootstrap.py +++ b/dspy/teleprompt/bootstrap.py @@ -1,16 +1,15 @@ -import dsp -import tqdm import random import threading -import dspy +import tqdm +import dsp +import dspy from dspy.primitives import Example from .teleprompt import Teleprompter from .vanilla import LabeledFewShot - # TODO: metrics should return an object with __bool__ basically, but fine if they're more complex. # They can also be sortable. diff --git a/dspy/teleprompt/finetune.py b/dspy/teleprompt/finetune.py index 29b7887ab5..8400fe7749 100644 --- a/dspy/teleprompt/finetune.py +++ b/dspy/teleprompt/finetune.py @@ -1,17 +1,18 @@ import os -import time -import dsp import random +import time import ujson from datasets.fingerprint import Hasher +import dsp from dspy.signatures.signature import signature_to_template -# from dspy.primitives import Example +from .bootstrap import BootstrapFewShot +# from dspy.primitives import Example from .teleprompt import Teleprompter -from .bootstrap import BootstrapFewShot + # from .vanilla import LabeledFewShot # from dspy.evaluate.evaluate import Evaluate diff --git a/dspy/teleprompt/knn_fewshot.py b/dspy/teleprompt/knn_fewshot.py index 4bc447abd3..b999447078 100644 --- a/dspy/teleprompt/knn_fewshot.py +++ b/dspy/teleprompt/knn_fewshot.py @@ -1,9 +1,11 @@ -from typing import List import types +from typing import List + import dsp +from dspy.teleprompt import BootstrapFewShot from .teleprompt import Teleprompter -from dspy.teleprompt import BootstrapFewShot + class KNNFewShot(Teleprompter): def __init__(self, KNN, k: int, trainset: List[dsp.Example]): diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py index 3f7f2cd636..bc0e5ef61b 100644 --- a/dspy/teleprompt/random_search.py +++ b/dspy/teleprompt/random_search.py @@ -1,13 +1,11 @@ import random +from dspy.evaluate.evaluate import Evaluate from dspy.teleprompt.teleprompt import Teleprompter from .bootstrap import BootstrapFewShot from .vanilla import LabeledFewShot -from dspy.evaluate.evaluate import Evaluate - - # TODO: Don't forget dealing with the raw demos. # TODO: Deal with the (pretty common) case of having a metric for filtering and a separate metric for eval. # The metric itself may tell though by the presence of trace. diff --git a/dspy/teleprompt/signature_opt.py b/dspy/teleprompt/signature_opt.py index 3163acf3f2..d71aa69a7d 100644 --- a/dspy/teleprompt/signature_opt.py +++ b/dspy/teleprompt/signature_opt.py @@ -1,9 +1,10 @@ +from collections import defaultdict + import dsp import dspy -from dspy.teleprompt.teleprompt import Teleprompter -from dspy.signatures import Signature from dspy.evaluate.evaluate import Evaluate -from collections import defaultdict +from dspy.signatures import Signature +from dspy.teleprompt.teleprompt import Teleprompter """ USAGE SUGGESTIONS: diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.py index 85298276ab..e316462c94 100644 --- a/dspy/teleprompt/signature_opt_bayesian.py +++ b/dspy/teleprompt/signature_opt_bayesian.py @@ -1,14 +1,16 @@ +import math +import random +from collections import defaultdict + +import optuna + import dsp import dspy -from dspy.signatures.signature import signature_to_template -from dspy.teleprompt.teleprompt import Teleprompter -from dspy.signatures import Signature from dspy.evaluate.evaluate import Evaluate -from collections import defaultdict -import random +from dspy.signatures import Signature +from dspy.signatures.signature import signature_to_template from dspy.teleprompt import BootstrapFewShot -import optuna -import math +from dspy.teleprompt.teleprompt import Teleprompter """ USAGE SUGGESTIONS: diff --git a/dspy/teleprompt/teleprompt_optuna.py b/dspy/teleprompt/teleprompt_optuna.py index 3686e66b7d..501bd71fde 100644 --- a/dspy/teleprompt/teleprompt_optuna.py +++ b/dspy/teleprompt/teleprompt_optuna.py @@ -1,10 +1,10 @@ import optuna +from dspy.evaluate.evaluate import Evaluate from dspy.teleprompt.teleprompt import Teleprompter from .bootstrap import BootstrapFewShot -from dspy.evaluate.evaluate import Evaluate class BootstrapFewShotWithOptuna(Teleprompter): def __init__(self, metric, teacher_settings={}, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1, num_candidate_programs=16, num_threads=6): diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.py index a0c997145b..81f31d94fb 100644 --- a/dspy/utils/dummies.py +++ b/dspy/utils/dummies.py @@ -1,9 +1,11 @@ import random -from dsp.modules import LM -from typing import List, Union, Dict +import re +from typing import Dict, List, Union + import numpy as np + +from dsp.modules import LM from dsp.utils.utils import dotdict -import re class DummyLM(LM): diff --git a/examples/longformqa/utils.py b/examples/longformqa/utils.py index e4ebadc829..bdae81a243 100644 --- a/examples/longformqa/utils.py +++ b/examples/longformqa/utils.py @@ -1,8 +1,10 @@ -import regex as re import nltk +import regex as re + nltk.download('punkt') from nltk.tokenize import sent_tokenize + def extract_text_by_citation(paragraph): citation_regex = re.compile(r'(.*?)(\[\d+\]\.)', re.DOTALL) parts_with_citation = citation_regex.findall(paragraph) diff --git a/inspect-app/app.py b/inspect-app/app.py index 5ea50dd92a..14ea405510 100644 --- a/inspect-app/app.py +++ b/inspect-app/app.py @@ -1,8 +1,9 @@ -from flask import Flask, request, jsonify -from flask_cors import CORS -import boto3 import time +import boto3 +from flask import Flask, jsonify, request +from flask_cors import CORS + app = Flask(__name__) CORS(app) diff --git a/setup.py b/setup.py index d31e9c7913..3f3bb18585 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup # Read the content of the README file with open('README.md', encoding='utf-8') as f: diff --git a/testing/optimizer_tester.py b/testing/optimizer_tester.py index e55f4dd4c4..68d3946019 100644 --- a/testing/optimizer_tester.py +++ b/testing/optimizer_tester.py @@ -1,18 +1,21 @@ -from .tasks.scone import ScoNeTask -from .tasks.hotpotqa import HotPotQATask -from .tasks.gsm8k import GSM8KTask -from .tasks.biodex import BioDexTask -from .tasks.tweet import TweetTask -from .tasks.tweet_metric import TweetMetricTask -from dspy.evaluate import Evaluate -from dotenv import load_dotenv -import openai -import os -import dspy import csv import datetime +import os from timeit import default_timer as timer +import openai +from dotenv import load_dotenv + +import dspy +from dspy.evaluate import Evaluate + +from .tasks.biodex import BioDexTask +from .tasks.gsm8k import GSM8KTask +from .tasks.hotpotqa import HotPotQATask +from .tasks.scone import ScoNeTask +from .tasks.tweet import TweetTask +from .tasks.tweet_metric import TweetMetricTask + datasets = ["ScoNe", "HotPotQA", "GSM8K", "BioDex", "Tweet"] class OptimizerTester: diff --git a/testing/tasks/__init__.py b/testing/tasks/__init__.py index f5034e36f8..0b1ee32ec9 100644 --- a/testing/tasks/__init__.py +++ b/testing/tasks/__init__.py @@ -1,7 +1,7 @@ from .base_task import BaseTask +from .biodex import BioDexTask from .gsm8k import GSM8KTask from .hotpotqa import HotPotQATask from .scone import ScoNeTask -from .biodex import BioDexTask from .tweet import TweetTask -from .tweet_metric import TweetMetricTask \ No newline at end of file +from .tweet_metric import TweetMetricTask diff --git a/testing/tasks/base_task.py b/testing/tasks/base_task.py index c7f8f42b68..089e6d7bee 100644 --- a/testing/tasks/base_task.py +++ b/testing/tasks/base_task.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod + class BaseTask(ABC): def __init__(self): pass diff --git a/testing/tasks/biodex.py b/testing/tasks/biodex.py index f98da11415..6e51745b37 100644 --- a/testing/tasks/biodex.py +++ b/testing/tasks/biodex.py @@ -1,20 +1,24 @@ from __future__ import annotations -from .base_task import BaseTask -import dspy -from dspy.evaluate import Evaluate -from dsp.utils import deduplicate -import tqdm -import datasets + import math -from functools import lru_cache import os -import re import pickle +import re +from collections import defaultdict +from enum import Enum +from functools import lru_cache +from typing import DefaultDict, Dict, List, Optional, Union + +import datasets +import tqdm from pydantic import BaseModel from rapidfuzz import process -from typing import Union, Optional, Dict, DefaultDict, List -from enum import Enum -from collections import defaultdict + +import dspy +from dsp.utils import deduplicate +from dspy.evaluate import Evaluate + +from .base_task import BaseTask # Must point to a MedDra download with mdhier.asc data_dir = '/future/u/okhattab/data/2023/MedDraV2/meddra_23_0_english/MedAscii' # NOTE: EDIT THIS LINE @@ -529,6 +533,7 @@ def __call__(self, gold, pred, trace=None): metricR = CompilationMetric(field='reactions', metric='recall') from collections import Counter + def reduce_grounded_reactions(grounded_reactions): scores = {} for score, reaction in grounded_reactions: diff --git a/testing/tasks/gsm8k.py b/testing/tasks/gsm8k.py index 6724ebdcd8..04f20d177c 100644 --- a/testing/tasks/gsm8k.py +++ b/testing/tasks/gsm8k.py @@ -1,11 +1,14 @@ -import dspy -from .base_task import BaseTask -from dspy.datasets.gsm8k import gsm8k_metric import random -import tqdm +import tqdm from datasets import load_dataset +import dspy +from dspy.datasets.gsm8k import gsm8k_metric + +from .base_task import BaseTask + + class CoT(dspy.Module): def __init__(self): super().__init__() diff --git a/testing/tasks/hotpotqa.py b/testing/tasks/hotpotqa.py index 19a5075fc9..f0bf17078e 100644 --- a/testing/tasks/hotpotqa.py +++ b/testing/tasks/hotpotqa.py @@ -1,7 +1,9 @@ import dspy +from dspy.datasets import HotPotQA from dspy.evaluate import Evaluate + from .base_task import BaseTask -from dspy.datasets import HotPotQA + class MultiHop(dspy.Module): def __init__(self,passages_per_hop): diff --git a/testing/tasks/scone.py b/testing/tasks/scone.py index 0ee5f0391f..27e8691acb 100644 --- a/testing/tasks/scone.py +++ b/testing/tasks/scone.py @@ -1,9 +1,13 @@ import glob import os +import random + import pandas as pd + import dspy + from .base_task import BaseTask -import random + def load_scone(dirname): dfs = [] diff --git a/testing/tasks/tweet.py b/testing/tasks/tweet.py index 30bdef68a5..73870a0591 100644 --- a/testing/tasks/tweet.py +++ b/testing/tasks/tweet.py @@ -1,10 +1,14 @@ -import dspy -from dspy.datasets import HotPotQA -from .base_task import BaseTask +import os from functools import lru_cache + import openai from dotenv import load_dotenv -import os + +import dspy +from dspy.datasets import HotPotQA + +from .base_task import BaseTask + class TweetSignature(dspy.Signature): ("""Given context and a question, answer with a tweet""") diff --git a/testing/tasks/tweet_metric.py b/testing/tasks/tweet_metric.py index baa7024ab7..b91458c48c 100644 --- a/testing/tasks/tweet_metric.py +++ b/testing/tasks/tweet_metric.py @@ -1,14 +1,18 @@ -import dspy -from dspy.datasets import HotPotQA -from .base_task import BaseTask -from dspy import Example +import os +import uuid from functools import lru_cache + import openai from dotenv import load_dotenv -import os -import uuid from tqdm import tqdm +import dspy +from dspy import Example +from dspy.datasets import HotPotQA + +from .base_task import BaseTask + + class TweetSignature(dspy.Signature): ("""Given context and a question, answer with a tweet""") From 1847fd39253a6d6fa8a898411754c96059987d01 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 11:16:16 -0600 Subject: [PATCH 09/21] Try and fix caching --- .github/workflows/run_tests.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 0513fe0477..4721e8f20a 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -34,23 +34,25 @@ jobs: python-version: ["3.9"] steps: - uses: actions/checkout@v3 - - name: Set up python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} + - name: Cache Poetry virtual environment uses: actions/cache@v3 + id: cached-poetry-dependencies with: path: ~/.cache/pypoetry key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | ${{ runner.os }}-poetry- - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install poetry + run: pipx install poetry + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - version: ${{ env.POETRY_VERSION }} + python-version: ${{ matrix.python-version }} + cache: "poetry" - name: Install dependencies - run: poetry install + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root - name: Run tests with pytest run: poetry run pytest tests/ @@ -63,18 +65,19 @@ jobs: python-version: ["3.9"] steps: - uses: actions/checkout@v3 + - name: Install poetry + run: pipx install poetry - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: "poetry" - uses: actions/cache@v3 with: path: ~/.cache/pypoetry key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | ${{ runner.os }}-poetry- - - name: Install Poetry - uses: snok/install-poetry@v1 - name: Build run: poetry build - name: Install built package @@ -93,17 +96,18 @@ jobs: python-version: ["3.9"] steps: - uses: actions/checkout@v3 + - name: Install poetry + run: pipx install poetry - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: "poetry" - uses: actions/cache@v3 with: path: ~/.cache/pypoetry key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | ${{ runner.os }}-poetry- - - name: Install Poetry - uses: snok/install-poetry@v1 - name: Run setup.py build run: python setup.py build From 6b9dbf9f53ebf2d2aec4f6bd72dfd9553a9d61f4 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 11:21:27 -0600 Subject: [PATCH 10/21] Fix import --- dspy/datasets/dataloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspy/datasets/dataloader.py b/dspy/datasets/dataloader.py index f4d3b87580..c9da0270db 100644 --- a/dspy/datasets/dataloader.py +++ b/dspy/datasets/dataloader.py @@ -5,7 +5,7 @@ from datasets import load_dataset import dspy -from dspy.datasets import Dataset +from dspy.datasets.dataset import Dataset class DataLoader(Dataset): From f2353c828de294627b155db976572705bc579ec2 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 11:25:16 -0600 Subject: [PATCH 11/21] Use natve python dep caching --- .github/workflows/run_tests.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4721e8f20a..97f3999e77 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -34,15 +34,6 @@ jobs: python-version: ["3.9"] steps: - uses: actions/checkout@v3 - - - name: Cache Poetry virtual environment - uses: actions/cache@v3 - id: cached-poetry-dependencies - with: - path: ~/.cache/pypoetry - key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry- - name: Install poetry run: pipx install poetry - name: Set up python ${{ matrix.python-version }} @@ -51,7 +42,6 @@ jobs: python-version: ${{ matrix.python-version }} cache: "poetry" - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root - name: Run tests with pytest run: poetry run pytest tests/ @@ -72,12 +62,6 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: "poetry" - - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry - key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry- - name: Build run: poetry build - name: Install built package @@ -103,11 +87,5 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: "poetry" - - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry - key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry- - name: Run setup.py build run: python setup.py build From 20e21db57c9fa5e806c200a9db191432a4a2707b Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 11:32:26 -0600 Subject: [PATCH 12/21] Update actions/checkout to v4 --- .github/workflows/run_tests.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 97f3999e77..3175b250d6 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -33,7 +33,9 @@ jobs: matrix: python-version: ["3.9"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} - name: Install poetry run: pipx install poetry - name: Set up python ${{ matrix.python-version }} @@ -54,7 +56,9 @@ jobs: matrix: python-version: ["3.9"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} - name: Install poetry run: pipx install poetry - name: Set up python ${{ matrix.python-version }} @@ -79,7 +83,9 @@ jobs: matrix: python-version: ["3.9"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} - name: Install poetry run: pipx install poetry - name: Set up python ${{ matrix.python-version }} From 3aed4d680a35e82b0f6d6855f2f351a30dee42bb Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 11:42:48 -0600 Subject: [PATCH 13/21] Use autofix in tests --- .github/workflows/run_tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 3175b250d6..a592457f88 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -45,6 +45,10 @@ jobs: cache: "poetry" - name: Install dependencies run: poetry install --no-interaction --no-root + - name: Run lint with tests + uses: chartboost/ruff-action@v1 + with: + args: --fix-only - name: Run tests with pytest run: poetry run pytest tests/ From 3d1a808c3f7d3ca5249fbc59e8ed8565521dada7 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 11:52:54 -0600 Subject: [PATCH 14/21] Try to cache "install poetry" --- .github/workflows/run_tests.yml | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index a592457f88..721771fd10 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -36,8 +36,15 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} - - name: Install poetry - run: pipx install poetry + - name: Load cached Poetry installation + id: cached-poetry + uses: actions/cache@v3 + with: + path: ~/.local + key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }} + - name: Install Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' + uses: snok/install-poetry@v1 - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -63,8 +70,15 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} - - name: Install poetry - run: pipx install poetry + - name: Load cached Poetry installation + id: cached-poetry + uses: actions/cache@v3 + with: + path: ~/.local + key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }} + - name: Install Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' + uses: snok/install-poetry@v1 - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -90,8 +104,15 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} - - name: Install poetry - run: pipx install poetry + - name: Load cached Poetry installation + id: cached-poetry + uses: actions/cache@v3 + with: + path: ~/.local + key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }} + - name: Install Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' + uses: snok/install-poetry@v1 - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From e3ddb1dc5aa93efb1c088d7a32251958fb79f63a Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 11:58:40 -0600 Subject: [PATCH 15/21] Empty-Commit to test caching From 382570dd53187aa1fb32cf14356a7d19f8c4b454 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 12:01:23 -0600 Subject: [PATCH 16/21] Remove workflow deps --- .github/workflows/run_tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 721771fd10..0cfc61d174 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -27,7 +27,6 @@ jobs: test: name: Run Tests - needs: lint runs-on: ubuntu-latest strategy: matrix: @@ -61,7 +60,6 @@ jobs: build_poetry: name: Build Poetry - needs: test runs-on: ubuntu-latest strategy: matrix: @@ -95,7 +93,6 @@ jobs: build_setup: name: Build Setup - needs: test runs-on: ubuntu-latest strategy: matrix: From a50b993a8b918c801831d1f9bc7a1544252e5a99 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 20:26:48 -0600 Subject: [PATCH 17/21] Rename action --- .github/workflows/run_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 0cfc61d174..4f2b13f43a 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -10,8 +10,8 @@ env: POETRY_VERSION: "1.6.1" jobs: - lint: - name: Lint + fix: + name: Apply Ruff Fix runs-on: ubuntu-latest permissions: contents: write From 423f1219f943a0e3c5a52da6d658bee4d1755499 Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 20:27:27 -0600 Subject: [PATCH 18/21] Update workflow name --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4f2b13f43a..52ac21e3c5 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -1,4 +1,4 @@ -name: Lint, Commit, and Test +name: Fix, Test, and Build on: push: From 98b982cb91eee8c630078d6c8203292f944a2b3e Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 21:29:59 -0600 Subject: [PATCH 19/21] Fix Docker stop command and subprocess run check --- dsp/modules/hf_client.py | 2 +- dsp/primitives/compiler.py | 2 +- dspy/functional/functional.py | 5 ++--- dspy/signatures/signature.py | 6 ++---- dspy/utils/dummies.py | 8 ++++---- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py index 428b0d06e6..fb92c48d53 100644 --- a/dsp/modules/hf_client.py +++ b/dsp/modules/hf_client.py @@ -173,7 +173,7 @@ def close_server(self, port): container_id = match.group(1) port_mapping = subprocess.check_output(['docker', 'port', container_id]).decode().strip() if f'0.0.0.0:{port}' in port_mapping: - subprocess.run(['docker', 'stop', container_id]) + subprocess.run(['docker', 'stop', container_id], check=False) def run_server(self, port, model_name=None, model_path=None, env_variable=None, gpus="all", num_shard=1, max_input_length=4000, max_total_tokens=4096, max_best_of=100): self.close_server(port) diff --git a/dsp/primitives/compiler.py b/dsp/primitives/compiler.py index e5bcd98b11..625ae23ac7 100644 --- a/dsp/primitives/compiler.py +++ b/dsp/primitives/compiler.py @@ -25,7 +25,7 @@ def openai_check_finetune(jobname): command = f"""openai api fine_tunes.get -i {jobname}""" print(command) - result = subprocess.run(command.split(), stdout=subprocess.PIPE) + result = subprocess.run(command.split(), stdout=subprocess.PIPE, check=False) output = result.stdout.decode("utf-8").strip() try: diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index 200f2c2bee..db083c5850 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -1,8 +1,7 @@ -from collections import defaultdict import inspect import json -from dspy.primitives.prediction import Prediction +from dspy.primitives.prediction import Prediction from dspy.signatures.signature import ensure_signature, make_signature MAX_RETRIES = 3 @@ -191,7 +190,7 @@ def forward(self, **kwargs) -> dspy.Prediction: else: # If there are no errors, we return the parsed results return Prediction.from_completions( - {key: [r[key] for r in parsed_results] for key in signature.output_fields} + {key: [r[key] for r in parsed_results] for key in signature.output_fields}, ) raise ValueError( "Too many retries trying to get the correct output format. " + "Try simplifying the requirements.", diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py index a127320858..22bc6475d7 100644 --- a/dspy/signatures/signature.py +++ b/dspy/signatures/signature.py @@ -1,13 +1,11 @@ import ast -from copy import deepcopy +import re import typing from copy import deepcopy -from typing import Dict, Tuple, Type, Union +from typing import Any, Dict, Tuple, Type, Union # noqa: UP035 from pydantic import BaseModel, Field, create_model from pydantic.fields import FieldInfo -from typing import Any, Type, Union, Dict, Tuple # noqa: UP035 -import re import dsp from dspy.signatures.field import InputField, OutputField, new_to_old_field diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.py index 9605498acb..d97d59b5a8 100644 --- a/dspy/utils/dummies.py +++ b/dspy/utils/dummies.py @@ -1,12 +1,12 @@ import random +import re from typing import Union -from dsp.modules import LM + import numpy as np -from dsp.utils.utils import dotdict -import re -from typing import Dict, List, Union from dsp.modules import LM +from dsp.utils.utils import dotdict + class DummyLM(LM): """Dummy language model for unit testing purposes.""" From 8185b701ac21c3c91462286e754236d56f53caab Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 21:43:22 -0600 Subject: [PATCH 20/21] Fix functional imports --- dspy/functional/functional.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index db083c5850..97bbdb8d89 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -1,6 +1,12 @@ import inspect import json +import typing +from typing import Annotated, List, Tuple +import pydantic + +import dspy +from dsp.templates.utils import passages2text from dspy.primitives.prediction import Prediction from dspy.signatures.signature import ensure_signature, make_signature From 278119388a9c9833594a4e23e6cc9e72a965ab9c Mon Sep 17 00:00:00 2001 From: Isaac Miller Date: Sun, 3 Mar 2024 23:07:45 -0600 Subject: [PATCH 21/21] Remove --no-root flag --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 52ac21e3c5..6b4221ade7 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -50,7 +50,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: "poetry" - name: Install dependencies - run: poetry install --no-interaction --no-root + run: poetry install --no-interaction - name: Run lint with tests uses: chartboost/ruff-action@v1 with: