diff --git a/docs/source/using-the-python-api.mdx b/docs/source/using-the-python-api.mdx index 1a21ebe4b..f09802272 100644 --- a/docs/source/using-the-python-api.mdx +++ b/docs/source/using-the-python-api.mdx @@ -12,9 +12,9 @@ import lighteval from lighteval.logging.evaluation_tracker import EvaluationTracker from lighteval.models.vllm.vllm_model import VLLMModelConfig from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters -from lighteval.utils.imports import is_accelerate_available +from lighteval.utils.imports import is_package_available -if is_accelerate_available(): +if is_package_available("accelerate"): from datetime import timedelta from accelerate import Accelerator, InitProcessGroupKwargs accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))]) diff --git a/pyproject.toml b/pyproject.toml index 75fff7cd6..24631db9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ dependencies = [ "fsspec>=2023.12.2", "httpx>=0.27.2", "latex2sympy2_extended==1.0.6", + "langcodes", ] [project.optional-dependencies] @@ -98,6 +99,7 @@ nanotron = [ ] tensorboardX = ["tensorboardX"] vllm = ["vllm>=0.10.0,<0.10.2", "ray", "more_itertools"] +sglang = ["sglang"] quality = ["ruff>=v0.11.0","pre-commit"] tests = ["pytest>=7.4.0","deepdiff","pip>=25.2"] dev = ["lighteval[accelerate,quality,tests,multilingual,math,extended_tasks,vllm]"] diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 108877601..aed32d2f1 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -43,13 +43,13 @@ TaskConfigLogger, VersionsLogger, ) -from lighteval.utils.imports import NO_TENSORBOARDX_WARN_MSG, is_nanotron_available, is_tensorboardX_available +from lighteval.utils.imports import is_package_available, not_installed_error_message from lighteval.utils.utils import obj_to_markdown logger = logging.getLogger(__name__) -if is_nanotron_available(): +if is_package_available("nanotron"): from nanotron.config import GeneralArgs # type: ignore try: @@ -659,11 +659,11 @@ def recreate_metadata_card(self, repo_id: str) -> None: # noqa: C901 def push_to_tensorboard( # noqa: C901 self, results: dict[str, dict[str, float]], details: dict[str, DetailsLogger.CompiledDetail] ): - if not is_tensorboardX_available: - logger.warning(NO_TENSORBOARDX_WARN_MSG) + if not is_package_available("tensorboardX"): + logger.warning(not_installed_error_message("tensorboardX")) return - if not is_nanotron_available(): + if not is_package_available("nanotron"): logger.warning("You cannot push results to tensorboard without having nanotron installed. Skipping") return diff --git a/src/lighteval/logging/info_loggers.py b/src/lighteval/logging/info_loggers.py index 64019ecf8..4482fabb2 100644 --- a/src/lighteval/logging/info_loggers.py +++ b/src/lighteval/logging/info_loggers.py @@ -34,13 +34,13 @@ from lighteval.models.model_output import ModelResponse from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig from lighteval.tasks.requests import Doc -from lighteval.utils.imports import is_nanotron_available +from lighteval.utils.imports import is_package_available logger = logging.getLogger(__name__) -if is_nanotron_available(): +if is_package_available("nanotron"): pass diff --git a/src/lighteval/main_nanotron.py b/src/lighteval/main_nanotron.py index 1131aea33..b844a74a4 100644 --- a/src/lighteval/main_nanotron.py +++ b/src/lighteval/main_nanotron.py @@ -32,11 +32,13 @@ reasoning_tags, remove_reasoning_tags, ) +from lighteval.utils.imports import requires SEED = 1234 +@requires("nanotron") def nanotron( checkpoint_config_path: Annotated[ str, Option(help="Path to the nanotron checkpoint YAML or python config file, potentially on s3.") @@ -45,12 +47,9 @@ def nanotron( remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, reasoning_tags: reasoning_tags.type = reasoning_tags.default, ): - """Evaluate models using nanotron as backend.""" - from lighteval.utils.imports import NO_NANOTRON_ERROR_MSG, is_nanotron_available - - if not is_nanotron_available(): - raise ImportError(NO_NANOTRON_ERROR_MSG) - + """ + Evaluate models using nanotron as backend. + """ from nanotron.config import GeneralArgs, ModelArgs, TokenizerArgs, get_config_from_dict, get_config_from_file from lighteval.logging.evaluation_tracker import EvaluationTracker diff --git a/src/lighteval/metrics/imports/data_stats_metric.py b/src/lighteval/metrics/imports/data_stats_metric.py index 883966be4..a007bcf5d 100644 --- a/src/lighteval/metrics/imports/data_stats_metric.py +++ b/src/lighteval/metrics/imports/data_stats_metric.py @@ -30,7 +30,7 @@ from typing import Literal from lighteval.metrics.imports.data_stats_utils import Fragments -from lighteval.utils.imports import NO_SPACY_ERROR_MSG, is_spacy_available +from lighteval.utils.imports import Extra, requires logger = logging.getLogger(__name__) @@ -55,6 +55,7 @@ def find_ngrams(input_list, n): return zip(*[input_list[i:] for i in range(n)]) +@requires(Extra.MULTILINGUAL) class DataStatsMetric(Metric): def __init__( self, @@ -86,8 +87,6 @@ def __init__( determines the spaCy model used for tokenization. Currently supports English, German, French, and Italian. """ - if not is_spacy_available(): - raise ImportError(NO_SPACY_ERROR_MSG) import spacy self.n_gram = n_gram diff --git a/src/lighteval/metrics/normalizations.py b/src/lighteval/metrics/normalizations.py index 925448f17..ef55681b1 100644 --- a/src/lighteval/metrics/normalizations.py +++ b/src/lighteval/metrics/normalizations.py @@ -28,6 +28,7 @@ from typing import Callable from lighteval.metrics.utils.linguistic_tokenizers import get_word_tokenizer +from lighteval.utils.imports import Extra, requires from lighteval.utils.language import Language @@ -444,15 +445,16 @@ def remove_punc(text: str) -> str: return "".join(ch for ch in text if ch not in PUNCT) +@requires(Extra.MULTILINGUAL) def get_multilingual_normalizer(lang: Language, lower: bool = True) -> Callable[[str], str]: """Get a normalizer function for the specified language. Returns: Callable[[str], str]: A function that normalizes text for the specified language """ - tokenizer = get_word_tokenizer(lang) def _inner_normalizer(text: str) -> str: + tokenizer = get_word_tokenizer(lang) text = remove_articles(text, lang) text = remove_punc(text) if lower: diff --git a/src/lighteval/metrics/utils/extractive_match_utils.py b/src/lighteval/metrics/utils/extractive_match_utils.py index d16145aea..cce2b1793 100644 --- a/src/lighteval/metrics/utils/extractive_match_utils.py +++ b/src/lighteval/metrics/utils/extractive_match_utils.py @@ -34,12 +34,12 @@ from lighteval.tasks.requests import Doc from lighteval.tasks.templates.utils.formulation import ChoicePrefix, get_prefix from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.imports import requires_latex2sympy2_extended +from lighteval.utils.imports import requires from lighteval.utils.language import Language from lighteval.utils.timeout import timeout -@requires_latex2sympy2_extended +@requires("latex2sympy2_extended") def latex_normalization_config_default_factory(): from latex2sympy2_extended.latex2sympy2 import NormalizationConfig @@ -373,7 +373,7 @@ def get_target_type_order(target_type: ExtractionTarget) -> int: # Small cache, to catche repeated calls invalid parsing @lru_cache(maxsize=20) -@requires_latex2sympy2_extended +@requires("latex2sympy2_extended") def parse_latex_with_timeout(latex: str, timeout_seconds: int): from latex2sympy2_extended.latex2sympy2 import latex2sympy @@ -428,7 +428,7 @@ def convert_to_pct(number: Number): return sympy.Mul(number, sympy.Rational(1, 100), evaluate=False) -@requires_latex2sympy2_extended +@requires("latex2sympy2_extended") @lru_cache(maxsize=20) def extract_latex( match: re.Match, latex_config: LatexExtractionConfig, timeout_seconds: int diff --git a/src/lighteval/metrics/utils/linguistic_tokenizers.py b/src/lighteval/metrics/utils/linguistic_tokenizers.py index e0dd9ef1a..a5670a268 100644 --- a/src/lighteval/metrics/utils/linguistic_tokenizers.py +++ b/src/lighteval/metrics/utils/linguistic_tokenizers.py @@ -18,10 +18,8 @@ from typing import Callable, Iterator from lighteval.utils.imports import ( - NO_SPACY_TOKENIZER_ERROR_MSG, - NO_STANZA_TOKENIZER_ERROR_MSG, - can_load_spacy_tokenizer, - can_load_stanza_tokenizer, + Extra, + requires, ) from lighteval.utils.language import Language @@ -99,11 +97,10 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]: return list(self.tokenizer.span_tokenize(text)) +@requires(Extra.MULTILINGUAL) class SpaCyTokenizer(WordTokenizer): def __init__(self, spacy_language: str, config=None): super().__init__() - if not can_load_spacy_tokenizer(spacy_language): - raise ImportError(NO_SPACY_TOKENIZER_ERROR_MSG) self.spacy_language = spacy_language self.config = config self._tokenizer = None @@ -137,11 +134,10 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]: ] +@requires("stanza") class StanzaTokenizer(WordTokenizer): def __init__(self, stanza_language: str, **stanza_kwargs): super().__init__() - if not can_load_stanza_tokenizer(): - raise ImportError(NO_STANZA_TOKENIZER_ERROR_MSG) self.stanza_language = stanza_language self.stanza_kwargs = stanza_kwargs self._tokenizer = None diff --git a/src/lighteval/metrics/utils/llm_as_judge.py b/src/lighteval/metrics/utils/llm_as_judge.py index 22da4b3e3..7e1b775c9 100644 --- a/src/lighteval/metrics/utils/llm_as_judge.py +++ b/src/lighteval/metrics/utils/llm_as_judge.py @@ -34,7 +34,7 @@ from tqdm import tqdm from tqdm.asyncio import tqdm_asyncio -from lighteval.utils.imports import is_litellm_available, is_openai_available, is_vllm_available +from lighteval.utils.imports import raise_if_package_not_available from lighteval.utils.utils import as_list @@ -151,8 +151,7 @@ def __lazy_load_client(self): # noqa: C901 # Both "openai" and "tgi" backends use the OpenAI-compatible API # They are handled separately to allow for backend-specific validation and setup case "openai" | "tgi": - if not is_openai_available(): - raise RuntimeError("OpenAI backend is not available.") + raise_if_package_not_available("openai") if self.client is None: from openai import OpenAI @@ -162,13 +161,11 @@ def __lazy_load_client(self): # noqa: C901 return self.__call_api_parallel case "litellm": - if not is_litellm_available(): - raise RuntimeError("litellm is not available.") + raise_if_package_not_available("litellm") return self.__call_litellm case "vllm": - if not is_vllm_available(): - raise RuntimeError("vllm is not available.") + raise_if_package_not_available("vllm") if self.pipe is None: from vllm import LLM, SamplingParams from vllm.transformers_utils.tokenizer import get_tokenizer diff --git a/src/lighteval/metrics/utils/math_comparison.py b/src/lighteval/metrics/utils/math_comparison.py index 2650ee335..2329acfe0 100644 --- a/src/lighteval/metrics/utils/math_comparison.py +++ b/src/lighteval/metrics/utils/math_comparison.py @@ -51,7 +51,7 @@ from sympy.core.function import UndefinedFunction from sympy.core.relational import Relational -from lighteval.utils.imports import requires_latex2sympy2_extended +from lighteval.utils.imports import requires from lighteval.utils.timeout import timeout @@ -308,7 +308,7 @@ def is_equation(expr: Basic | MatrixBase) -> bool: return False -@requires_latex2sympy2_extended +@requires("latex2sympy2_extended") def is_assignment_relation(expr: Basic | MatrixBase) -> bool: from latex2sympy2_extended.latex2sympy2 import is_expr_of_only_symbols diff --git a/src/lighteval/models/endpoints/litellm_model.py b/src/lighteval/models/endpoints/litellm_model.py index 00f7c8779..e620fba70 100644 --- a/src/lighteval/models/endpoints/litellm_model.py +++ b/src/lighteval/models/endpoints/litellm_model.py @@ -32,12 +32,12 @@ from lighteval.tasks.prompt_manager import PromptManager from lighteval.tasks.requests import Doc, SamplingMethod from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import is_litellm_available +from lighteval.utils.imports import is_package_available, requires logger = logging.getLogger(__name__) -if is_litellm_available(): +if is_package_available("litellm"): import litellm from litellm import encode from litellm.caching.caching import Cache @@ -110,6 +110,7 @@ class LiteLLMModelConfig(ModelConfig): concurrent_requests: int = 10 +@requires("litellm") class LiteLLMClient(LightevalModel): _DEFAULT_MAX_LENGTH: int = 4096 diff --git a/src/lighteval/models/endpoints/tgi_model.py b/src/lighteval/models/endpoints/tgi_model.py index 8130cba88..4fd765b8d 100644 --- a/src/lighteval/models/endpoints/tgi_model.py +++ b/src/lighteval/models/endpoints/tgi_model.py @@ -32,10 +32,10 @@ from lighteval.models.endpoints.endpoint_model import InferenceEndpointModel from lighteval.tasks.prompt_manager import PromptManager from lighteval.utils.cache_management import SampleCache -from lighteval.utils.imports import NO_TGI_ERROR_MSG, is_tgi_available +from lighteval.utils.imports import Extra, is_package_available, requires -if is_tgi_available(): +if is_package_available(Extra.TGI): from text_generation import AsyncClient else: from unittest.mock import Mock @@ -103,8 +103,6 @@ class ModelClient(InferenceEndpointModel): _DEFAULT_MAX_LENGTH: int = 4096 def __init__(self, config: TGIModelConfig) -> None: - if not is_tgi_available(): - raise ImportError(NO_TGI_ERROR_MSG) headers = ( {} if config.inference_server_auth is None else {"Authorization": f"Bearer {config.inference_server_auth}"} ) @@ -135,6 +133,7 @@ def __init__(self, config: TGIModelConfig) -> None: # Initialize cache for tokenization and predictions self._cache = SampleCache(config) + @requires(Extra.TGI) def _async_process_request( self, context: str, @@ -174,6 +173,7 @@ def _async_process_request( return generated_text + @requires(Extra.TGI) def _process_request(self, *args, **kwargs) -> TextGenerationOutput: return asyncio.run(self._async_process_request(*args, **kwargs)) diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py index ccae20a5f..e3c710edd 100644 --- a/src/lighteval/models/model_loader.py +++ b/src/lighteval/models/model_loader.py @@ -43,16 +43,6 @@ from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModel, VLMTransformersModelConfig from lighteval.models.vllm.vllm_model import AsyncVLLMModel, VLLMModel, VLLMModelConfig -from lighteval.utils.imports import ( - NO_LITELLM_ERROR_MSG, - NO_SGLANG_ERROR_MSG, - NO_TGI_ERROR_MSG, - NO_VLLM_ERROR_MSG, - is_litellm_available, - is_sglang_available, - is_tgi_available, - is_vllm_available, -) logger = logging.getLogger(__name__) @@ -102,18 +92,12 @@ def load_model( # noqa: C901 def load_model_with_tgi(config: TGIModelConfig): - if not is_tgi_available(): - raise ImportError(NO_TGI_ERROR_MSG) - logger.info(f"Load model from inference server: {config.inference_server_address}") model = ModelClient(config=config) return model def load_litellm_model(config: LiteLLMModelConfig): - if not is_litellm_available(): - raise ImportError(NO_LITELLM_ERROR_MSG) - model = LiteLLMClient(config) return model @@ -163,8 +147,6 @@ def load_model_with_accelerate_or_default( elif isinstance(config, DeltaModelConfig): model = DeltaModel(config=config) elif isinstance(config, VLLMModelConfig): - if not is_vllm_available(): - raise ImportError(NO_VLLM_ERROR_MSG) if config.is_async: model = AsyncVLLMModel(config=config) else: @@ -186,7 +168,4 @@ def load_inference_providers_model(config: InferenceProvidersModelConfig): def load_sglang_model(config: SGLangModelConfig): - if not is_sglang_available(): - raise ImportError(NO_SGLANG_ERROR_MSG) - return SGLangModel(config=config) diff --git a/src/lighteval/models/nanotron/nanotron_model.py b/src/lighteval/models/nanotron/nanotron_model.py index 310843d32..7ed6d35eb 100644 --- a/src/lighteval/models/nanotron/nanotron_model.py +++ b/src/lighteval/models/nanotron/nanotron_model.py @@ -51,7 +51,7 @@ SamplingMethod, ) from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import is_nanotron_available +from lighteval.utils.imports import is_package_available from lighteval.utils.parallelism import find_executable_batch_size from lighteval.utils.utils import as_list @@ -63,7 +63,7 @@ TokenSequence = Union[List[int], torch.LongTensor, torch.Tensor, BatchEncoding] -if is_nanotron_available(): +if is_package_available("nanotron"): from nanotron import distributed as dist from nanotron import logging from nanotron.config import GeneralArgs, ModelArgs, TokenizerArgs diff --git a/src/lighteval/models/sglang/sglang_model.py b/src/lighteval/models/sglang/sglang_model.py index fe37c64f9..e5c0f4d87 100644 --- a/src/lighteval/models/sglang/sglang_model.py +++ b/src/lighteval/models/sglang/sglang_model.py @@ -35,12 +35,12 @@ from lighteval.tasks.prompt_manager import PromptManager from lighteval.tasks.requests import Doc, SamplingMethod from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import is_sglang_available +from lighteval.utils.imports import is_package_available, requires logger = logging.getLogger(__name__) -if is_sglang_available(): +if is_package_available("sglang"): from sglang import Engine from sglang.srt.hf_transformers_utils import get_tokenizer @@ -186,7 +186,7 @@ def add_special_tokens(self): def max_length(self) -> int: return self._max_length - def _create_auto_model(self, config: SGLangModelConfig) -> Optional[Engine]: + def _create_auto_model(self, config: SGLangModelConfig) -> Optional["Engine"]: self.model_args = { "model_path": config.model_name, "trust_remote_code": config.trust_remote_code, @@ -313,6 +313,7 @@ def _greedy_until( results.append(cur_response) return dataset.get_original_order(results) + @requires("sglang") def _generate( self, inputs: list[list[int]], diff --git a/src/lighteval/models/transformers/adapter_model.py b/src/lighteval/models/transformers/adapter_model.py index a868ad20f..52f339664 100644 --- a/src/lighteval/models/transformers/adapter_model.py +++ b/src/lighteval/models/transformers/adapter_model.py @@ -30,15 +30,16 @@ from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig from lighteval.models.utils import _get_dtype -from lighteval.utils.imports import NO_PEFT_ERROR_MSG, is_peft_available +from lighteval.utils.imports import is_package_available, requires logger = logging.getLogger(__name__) -if is_peft_available(): +if is_package_available("peft"): from peft import PeftModel +@requires("peft") class AdapterModelConfig(TransformersModelConfig): """Configuration class for PEFT (Parameter-Efficient Fine-Tuning) adapter models. @@ -58,10 +59,6 @@ class AdapterModelConfig(TransformersModelConfig): base_model: str - def model_post_init(self, __context): - if not is_peft_available(): - raise ImportError(NO_PEFT_ERROR_MSG) - class AdapterModel(TransformersModel): def _create_auto_model(self) -> transformers.PreTrainedModel: diff --git a/src/lighteval/models/transformers/transformers_model.py b/src/lighteval/models/transformers/transformers_model.py index 0fb6df464..ed97faf84 100644 --- a/src/lighteval/models/transformers/transformers_model.py +++ b/src/lighteval/models/transformers/transformers_model.py @@ -55,7 +55,7 @@ from lighteval.tasks.requests import Doc, SamplingMethod from lighteval.utils.cache_management import SampleCache, cached from lighteval.utils.imports import ( - is_accelerate_available, + is_package_available, ) from lighteval.utils.parallelism import find_executable_batch_size @@ -227,7 +227,7 @@ def __init__( self.model_name = _simplify_name(config.model_name) - if is_accelerate_available(): + if is_package_available("accelerate"): model_size, _ = calculate_maximum_sizes(self.model) model_size = convert_bytes(model_size) else: @@ -290,7 +290,7 @@ def from_model( else: self._device = self.config.device - if is_accelerate_available(): + if is_package_available("accelerate"): model_size, _ = calculate_maximum_sizes(self.model) model_size = convert_bytes(model_size) else: @@ -331,7 +331,7 @@ def disable_tqdm(self) -> bool: def init_model_parallel(self, model_parallel: bool | None = None) -> Tuple[bool, Optional[dict], Optional[str]]: """Compute all the parameters related to model_parallel""" - if not is_accelerate_available(): + if not is_package_available("accelerate"): return False, None, None self.num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) diff --git a/src/lighteval/models/transformers/vlm_transformers_model.py b/src/lighteval/models/transformers/vlm_transformers_model.py index 3da1290be..0697ab729 100644 --- a/src/lighteval/models/transformers/vlm_transformers_model.py +++ b/src/lighteval/models/transformers/vlm_transformers_model.py @@ -47,7 +47,7 @@ from lighteval.tasks.requests import Doc, SamplingMethod from lighteval.utils.cache_management import SampleCache, cached from lighteval.utils.imports import ( - is_accelerate_available, + is_package_available, ) @@ -210,7 +210,7 @@ def disable_tqdm(self) -> bool: # Copied from ./transformers_model.py def init_model_parallel(self, model_parallel: bool | None = None) -> Tuple[bool, Optional[dict], Optional[str]]: """Compute all the parameters related to model_parallel""" - if not is_accelerate_available(): + if not is_package_available("accelerate"): return False, None, None self.num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) diff --git a/src/lighteval/models/vllm/vllm_model.py b/src/lighteval/models/vllm/vllm_model.py index 68632410a..f0a9123e9 100644 --- a/src/lighteval/models/vllm/vllm_model.py +++ b/src/lighteval/models/vllm/vllm_model.py @@ -38,13 +38,13 @@ from lighteval.tasks.prompt_manager import PromptManager from lighteval.tasks.requests import Doc, SamplingMethod from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import is_vllm_available +from lighteval.utils.imports import is_package_available, requires logger = logging.getLogger(__name__) -if is_vllm_available(): +if is_package_available("vllm"): import ray from more_itertools import distribute from vllm import LLM, RequestOutput, SamplingParams @@ -179,6 +179,7 @@ class VLLMModelConfig(ModelConfig): override_chat_template: bool = None +@requires("vllm") class VLLMModel(LightevalModel): def __init__( self, @@ -531,6 +532,7 @@ def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: raise NotImplementedError() +@requires("vllm") class AsyncVLLMModel(VLLMModel): """VLLM models which deploy async natively (no ray). Supports DP and PP/TP but not batch size > 1""" diff --git a/src/lighteval/pipeline.py b/src/lighteval/pipeline.py index f79e8c910..fab0ea7d4 100644 --- a/src/lighteval/pipeline.py +++ b/src/lighteval/pipeline.py @@ -42,31 +42,19 @@ from lighteval.tasks.lighteval_task import LightevalTask from lighteval.tasks.registry import Registry from lighteval.tasks.requests import SamplingMethod -from lighteval.utils.imports import ( - NO_ACCELERATE_ERROR_MSG, - NO_NANOTRON_ERROR_MSG, - NO_OPENAI_ERROR_MSG, - NO_SGLANG_ERROR_MSG, - NO_TGI_ERROR_MSG, - NO_VLLM_ERROR_MSG, - is_accelerate_available, - is_nanotron_available, - is_openai_available, - is_sglang_available, - is_tgi_available, - is_vllm_available, -) +from lighteval.utils.imports import is_package_available from lighteval.utils.parallelism import test_all_gather from lighteval.utils.utils import make_results_table, remove_reasoning_tags -if is_accelerate_available(): +if is_package_available("accelerate"): from accelerate import Accelerator, InitProcessGroupKwargs else: from unittest.mock import Mock Accelerator = InitProcessGroupKwargs = Mock() -if is_nanotron_available(): + +if is_package_available("nanotron"): from nanotron import distributed as dist from nanotron.parallel.context import ParallelContext @@ -108,27 +96,6 @@ class PipelineParameters: bootstrap_iters: int = 1000 def __post_init__(self): # noqa C901 - # Import testing - if self.launcher_type == ParallelismManager.ACCELERATE: - if not is_accelerate_available(): - raise ImportError(NO_ACCELERATE_ERROR_MSG) - elif self.launcher_type == ParallelismManager.VLLM: - if not is_vllm_available(): - raise ImportError(NO_VLLM_ERROR_MSG) - elif self.launcher_type == ParallelismManager.SGLANG: - if not is_sglang_available(): - raise ImportError(NO_SGLANG_ERROR_MSG) - elif self.launcher_type == ParallelismManager.TGI: - if not is_tgi_available(): - raise ImportError(NO_TGI_ERROR_MSG) - elif self.launcher_type == ParallelismManager.NANOTRON: - if not is_nanotron_available(): - raise ImportError(NO_NANOTRON_ERROR_MSG) - elif self.launcher_type == ParallelismManager.OPENAI: - if not is_openai_available(): - raise ImportError(NO_OPENAI_ERROR_MSG) - - # Convert reasoning tags to list if needed if not isinstance(self.reasoning_tags, list): try: self.reasoning_tags = ast.literal_eval(self.reasoning_tags) @@ -189,12 +156,12 @@ def __init__( def _init_parallelism_manager(self): accelerator, parallel_context = None, None if self.launcher_type == ParallelismManager.ACCELERATE: - if not is_accelerate_available(): + if not is_package_available("accelerate"): raise ValueError("You are trying to launch an accelerate model, but accelerate is not installed") accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))]) test_all_gather(accelerator=accelerator) elif self.launcher_type == ParallelismManager.NANOTRON: - if not is_nanotron_available(): + if not is_package_available("nanotron"): raise ValueError("You are trying to launch a nanotron model, but nanotron is not installed") dist.initialize_torch_distributed() parallel_context = ParallelContext( diff --git a/src/lighteval/tasks/extended/__init__.py b/src/lighteval/tasks/extended/__init__.py index cd60bdb1b..247a0c3a2 100644 --- a/src/lighteval/tasks/extended/__init__.py +++ b/src/lighteval/tasks/extended/__init__.py @@ -20,20 +20,15 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from lighteval.utils.imports import can_load_extended_tasks +import lighteval.tasks.extended.hle.main as hle +import lighteval.tasks.extended.ifbench.main as ifbench +import lighteval.tasks.extended.ifeval.main as ifeval +import lighteval.tasks.extended.lcb.main as lcb +import lighteval.tasks.extended.mix_eval.main as mix_eval +import lighteval.tasks.extended.mt_bench.main as mt_bench +import lighteval.tasks.extended.olympiade_bench.main as olympiad_bench +import lighteval.tasks.extended.tiny_benchmarks.main as tiny_benchmarks -if can_load_extended_tasks(): - import lighteval.tasks.extended.hle.main as hle - import lighteval.tasks.extended.ifbench.main as ifbench - import lighteval.tasks.extended.ifeval.main as ifeval - import lighteval.tasks.extended.lcb.main as lcb - import lighteval.tasks.extended.mix_eval.main as mix_eval - import lighteval.tasks.extended.mt_bench.main as mt_bench - import lighteval.tasks.extended.olympiade_bench.main as olympiad_bench - import lighteval.tasks.extended.tiny_benchmarks.main as tiny_benchmarks - AVAILABLE_EXTENDED_TASKS_MODULES = [ifeval, ifbench, tiny_benchmarks, mt_bench, mix_eval, olympiad_bench, hle, lcb] - -else: - AVAILABLE_EXTENDED_TASKS_MODULES = [] +AVAILABLE_EXTENDED_TASKS_MODULES = [ifeval, ifbench, tiny_benchmarks, mt_bench, mix_eval, olympiad_bench, hle, lcb] diff --git a/src/lighteval/tasks/extended/ifbench/instructions.py b/src/lighteval/tasks/extended/ifbench/instructions.py index ccb5b50da..0c4f0a9a0 100644 --- a/src/lighteval/tasks/extended/ifbench/instructions.py +++ b/src/lighteval/tasks/extended/ifbench/instructions.py @@ -25,8 +25,15 @@ import emoji import nltk -import spacy -import syllapy + +from lighteval.utils.imports import is_package_available, requires + + +if is_package_available("syllapy"): + import syllapy + +if is_package_available("spacy"): + import spacy import lighteval.tasks.extended.ifeval.instructions_utils as instructions_util @@ -61,6 +68,7 @@ RESOURCES_DOWNLOADED: bool = False +@requires("syllapy", "spacy") class Instruction: """An instruction template.""" diff --git a/src/lighteval/tasks/extended/ifeval/instructions.py b/src/lighteval/tasks/extended/ifeval/instructions.py index 4022e640f..06b7cf85c 100644 --- a/src/lighteval/tasks/extended/ifeval/instructions.py +++ b/src/lighteval/tasks/extended/ifeval/instructions.py @@ -21,7 +21,11 @@ import re import string -import langdetect +from lighteval.utils.imports import is_package_available + + +if is_package_available("langdetect"): + import langdetect import lighteval.tasks.extended.ifeval.instructions_utils as instructions_util diff --git a/src/lighteval/tasks/extended/ifeval/main.py b/src/lighteval/tasks/extended/ifeval/main.py index 1f63f91f1..ae7d42809 100644 --- a/src/lighteval/tasks/extended/ifeval/main.py +++ b/src/lighteval/tasks/extended/ifeval/main.py @@ -31,9 +31,11 @@ from lighteval.models.model_output import ModelResponse from lighteval.tasks.lighteval_task import LightevalTaskConfig from lighteval.tasks.requests import Doc, SamplingMethod +from lighteval.utils.imports import requires # Very specific task where there are no precise outputs but instead we test if the format obeys rules +@requires("langdetect") def ifeval_prompt(line, task_name: str = ""): return Doc( task_name=task_name, @@ -123,6 +125,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict: } +@requires("langdetect") def agg_inst_level_acc(items): flat_items = [item for sublist in items for item in sublist] inst_level_acc = sum(flat_items) / len(flat_items) diff --git a/src/lighteval/tasks/registry.py b/src/lighteval/tasks/registry.py index 01125c778..95914991c 100644 --- a/src/lighteval/tasks/registry.py +++ b/src/lighteval/tasks/registry.py @@ -36,12 +36,6 @@ import lighteval.tasks.default_tasks as default_tasks from lighteval.tasks.extended import AVAILABLE_EXTENDED_TASKS_MODULES from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig -from lighteval.utils.imports import ( - CANNOT_USE_EXTENDED_TASKS_MSG, - CANNOT_USE_MULTILINGUAL_TASKS_MSG, - can_load_extended_tasks, - can_load_multilingual_tasks, -) # Import community tasks @@ -220,12 +214,8 @@ def _activate_loading_of_optional_suite(self) -> None: ) if "extended" in suites: - if not can_load_extended_tasks(): - raise ImportError(CANNOT_USE_EXTENDED_TASKS_MSG) self._load_extended = True if "multilingual" in suites: - if not can_load_multilingual_tasks(): - raise ImportError(CANNOT_USE_MULTILINGUAL_TASKS_MSG) self._load_multilingual = True if "community" in suites: self._load_community = True @@ -251,8 +241,6 @@ def _load_full_registry(self) -> dict[str, LightevalTaskConfig]: if self._load_extended: for extended_task_module in AVAILABLE_EXTENDED_TASKS_MODULES: custom_tasks_module.append(extended_task_module) - else: - logger.warning(CANNOT_USE_EXTENDED_TASKS_MSG) # Need to load community tasks if self._load_community: diff --git a/src/lighteval/utils/imports.py b/src/lighteval/utils/imports.py index 2534cb52a..048526cbc 100644 --- a/src/lighteval/utils/imports.py +++ b/src/lighteval/utils/imports.py @@ -11,158 +11,206 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import enum +import functools import importlib +import inspect +import re +from collections import defaultdict +from functools import lru_cache +from importlib.metadata import PackageNotFoundError, metadata, version +from typing import Dict, List, Tuple + +from packaging.requirements import Requirement +from packaging.version import Version + + +# These extras should exist in the pyproject.toml file +class Extra(enum.Enum): + MULTILINGUAL = "multilingual" + EXTENDED = "extended" + TGI = "tgi" + + +@lru_cache() +def is_package_available(package: str | Requirement | Extra): + """ + Check if a package is installed in the environment. Returns True if that's the case, False otherwise. + """ + deps, deps_by_extra = required_dependencies() + + # If the package is a string, it will get the potential required version from the pyproject.toml + if isinstance(package, str): + if package not in deps: + raise RuntimeError( + f"Package {package} was tested against, but isn't specified in the pyproject.toml file. Please specify" + f"it as a potential dependency or an extra for it to be checked." + ) + package = deps[package] + + # If the specified package is an "Extra", we will iterate through each required dependency of that extra + # and their version and check their existence. + if isinstance(package, Extra): + dependencies = deps_by_extra[package.value] + return all(is_package_available(_package) for _package in dependencies) + else: + try: + installed = Version(version(package.name)) + except PackageNotFoundError: + return False + + # No version constraint → any installed version is OK + if not package.specifier: + return True + + return installed in package.specifier + + +@lru_cache() +def required_dependencies() -> Tuple[Dict[str, Requirement], Dict[str, List[Requirement]]]: + """ + Parse the pyproject.toml file and return a dictionary mapping package names to requirements. + """ + md = metadata("lighteval") + requires_dist = md.get_all("Requires-Dist") or [] + deps_by_extra = defaultdict(list) + deps = {} + + for dep in requires_dist: + extra = None + if ";" in dep: + dep, marker = dep.split(";", 1) + + # The `metadata` function prints requirements as follows + # 'vllm<0.10.2,>=0.10.0; extra == "vllm"' + # The regex searches for "extra == " in order to parse the marker. + match = re.search(r'extra\s*==\s*"(.*?)"', marker) + extra = match.group(1) if match else None + requirement = Requirement(dep.strip()) + deps_by_extra[extra].append(requirement) + deps[requirement.name] = requirement + + return deps, deps_by_extra + + +@lru_cache() +def is_multilingual_package_available(language: str): + imports = [] + packages = ["spacy", "stanza"] + if language == "vi": + packages.append("pyvi") + elif language == "zh": + packages.append("jieba") + for package in packages: + imports.append(importlib.util.find_spec(package)) -def is_accelerate_available() -> bool: - return importlib.util.find_spec("accelerate") is not None - - -NO_ACCELERATE_ERROR_MSG = "You requested the use of accelerate for this evaluation, but it is not available in your current environement. Please install it using pip." - - -def is_tgi_available() -> bool: - return importlib.util.find_spec("text_generation") is not None - - -NO_TGI_ERROR_MSG = "You are trying to start a text generation inference endpoint, but text-generation is not present in your local environement. Please install it using pip." - - -def is_nanotron_available() -> bool: - return importlib.util.find_spec("nanotron") is not None - - -NO_NANOTRON_ERROR_MSG = "You requested the use of nanotron for this evaluation, but it is not available in your current environement. Please install it using pip." - - -def is_optimum_available() -> bool: - return importlib.util.find_spec("optimum") is not None - - -def is_bnb_available() -> bool: - return importlib.util.find_spec("bitsandbytes") is not None - - -NO_BNB_ERROR_MSG = "You are trying to load a model quantized with `bitsandbytes`, which is not available in your local environement. Please install it using pip." - - -def is_autogptq_available() -> bool: - return importlib.util.find_spec("auto_gptq") is not None - - -NO_AUTOGPTQ_ERROR_MSG = "You are trying to load a model quantized with `auto-gptq`, which is not available in your local environement. Please install it using pip." - - -def is_peft_available() -> bool: - return importlib.util.find_spec("peft") is not None - - -NO_PEFT_ERROR_MSG = "You are trying to use adapter weights models, for which you need `peft`, which is not available in your environment. Please install it using pip." - - -def is_tensorboardX_available() -> bool: - return importlib.util.find_spec("tensorboardX") is not None - - -NO_TENSORBOARDX_WARN_MSG = ( - "You are trying to log using tensorboardX, which is not installed. Please install it using pip. Skipping." -) - - -def is_openai_available() -> bool: - return importlib.util.find_spec("openai") is not None - - -NO_OPENAI_ERROR_MSG = "You are trying to use an Open AI LLM as a judge, for which you need `openai`, which is not available in your environment. Please install it using pip." - - -def is_litellm_available() -> bool: - return importlib.util.find_spec("litellm") is not None - - -NO_LITELLM_ERROR_MSG = "You are trying to use a LiteLLM model, for which you need `litellm`, which is not available in your environment. Please install it using pip." - - -def is_vllm_available() -> bool: - return importlib.util.find_spec("vllm") is not None and importlib.util.find_spec("ray") is not None - - -NO_VLLM_ERROR_MSG = "You are trying to use an VLLM model, for which you need `vllm` and `ray`, which are not available in your environment. Please install them using pip, `pip install vllm ray`." - - -def is_sglang_available() -> bool: - return importlib.util.find_spec("sglang") is not None and importlib.util.find_spec("flashinfer") is not None - + return all(cur_import is not None for cur_import in imports) -NO_SGLANG_ERROR_MSG = "You are trying to use an sglang model, for which you need `sglang` and `flashinfer`, which are not available in your environment. Please install them using pip, `pip install vllm ray`." +def raise_if_package_not_available(package: Requirement | Extra, *, language: str = None, object_name: str = None): + prefix = "You" if object_name is None else f"Through the use of {object_name}, you" -def can_load_extended_tasks() -> bool: - imports = [] - for package in ["langdetect", "openai"]: - imports.append(importlib.util.find_spec(package)) + if package == Extra.MULTILINGUAL and ((language is not None) or (not is_multilingual_package_available(language))): + raise ImportError(prefix + not_installed_error_message(package)[3:]) - return all(cur_import is not None for cur_import in imports) + if not is_package_available(package): + raise ImportError(prefix + not_installed_error_message(package)[3:]) -CANNOT_USE_EXTENDED_TASKS_MSG = "If you want to use extended_tasks, make sure you installed their dependencies using `pip install -e .[extended_tasks]`." +def not_installed_error_message(package: Requirement) -> str: + """ + Custom error messages if need be. + """ + if package == Extra.MULTILINGUAL.value: + return "You are trying to run an evaluation requiring multilingual capabilities. Please install the required extra: `pip install lighteval[multilingual]`" + elif package == Extra.EXTENDED.value: + return "You are trying to run an evaluation requiring additional extensions. Please install the required extra: `pip install lighteval[extended] " + elif package == "text_generation": + return "You are trying to start a text generation inference endpoint, but TGI is not present in your local environment. Please install it using pip." + elif package == "peft": + return "You are trying to use adapter weights models, for which you need `peft`, which is not available in your environment. Please install it using pip." + elif package == "openai": + return "You are trying to use an Open AI LLM as a judge, for which you need `openai`, which is not available in your environment. Please install it using pip." -def can_load_multilingual_tasks() -> bool: - try: - import lighteval.tasks.multilingual.tasks # noqa: F401 + if isinstance(package, Extra): + return f"You are trying to run an evaluation requiring {package.value} capabilities. Please install the required extra: `pip install lighteval[{package.value}]`" + else: + return f"You requested the use of `{package}` for this evaluation, but it is not available in your current environment. Please install it using pip." - return True - except ImportError: - return False +class DummyObject(type): + """ + Metaclass for the dummy objects. Any class inheriting from it will return the ImportError generated by + `requires_backend` each time a user tries to access any method of that class. + """ -CANNOT_USE_MULTILINGUAL_TASKS_MSG = "If you want to use multilingual tasks, make sure you installed their dependencies using `pip install -e .[multilingual]`." + is_dummy = True + def __getattribute__(cls, key): + if (key.startswith("_") and key != "_from_config") or key == "is_dummy" or key == "mro" or key == "call": + return super().__getattribute__(key) -def can_load_spacy_tokenizer(language: str) -> bool: - imports = [] - packages = ["spacy", "stanza"] - if language == "vi": - packages.append("pyvi") - elif language == "zh": - packages.append("jieba") + for backend in cls._backends: + raise_if_package_not_available(backend) - for package in packages: - imports.append(importlib.util.find_spec(package)) - return all(cur_import is not None for cur_import in imports) + return super().__getattribute__(key) -NO_SPACY_TOKENIZER_ERROR_MSG = "You are trying to load a spacy tokenizer, for which you need `spacy` and its dependencies, which are not available in your environment. Please install them using `pip install lighteval[multilingual]`." +def parse_specified_backends(specified_backends): + requirements, _ = required_dependencies() + applied_backends = [] + for backend in specified_backends: + if isinstance(backend, Extra): + applied_backends.append(backend if isinstance(backend, Extra) else requirements[backend]) + elif backend not in requirements: + raise RuntimeError( + "A dependency was specified with @requires, but it is not defined in the possible dependencies " + f"defined in the pyproject.toml: `{backend}`." + f"" + f"If editing the pyproject.toml to add a new dependency, remember to reinstall lighteval for the" + f"update to take effect." + ) + else: + applied_backends.append(requirements[backend]) -def can_load_stanza_tokenizer() -> bool: - return importlib.util.find_spec("stanza") is not None + return applied_backends -NO_STANZA_TOKENIZER_ERROR_MSG = "You are trying to load a stanza tokenizer, for which you need `stanza`, which is not available in your environment. Please install it using `pip install lighteval[multilingual]`." +def requires(*specified_backends): + """ + Decorator to raise an ImportError if the decorated object (function or class) requires a dependency + which is not installed. + """ + applied_backends = parse_specified_backends(specified_backends) -# Better than having to check import every time -def requires_latex2sympy2_extended(func): - checked_import = False + def inner_fn(_object): + _object._backends = applied_backends - def wrapper(*args, **kwargs): - nonlocal checked_import - if not checked_import and importlib.util.find_spec("latex2sympy2_extended") is None: - raise ImportError(NO_LATEX2SYMPY2_EXTENDED_ERROR_MSG) - checked_import = True - return func(*args, **kwargs) + if inspect.isclass(_object): - return wrapper + class Placeholder(metaclass=DummyObject): + _backends = applied_backends + def __init__(self, *args, **kwargs): + for backend in self._backends: + raise_if_package_not_available(backend, object_name=_object.__name__) -NO_LATEX2SYMPY2_EXTENDED_ERROR_MSG = "You are trying to parse latex expressions, for which you need `latex2sympy2_extended`, which is not available in your environment. Please install it using `pip install lighteval[math]`." + Placeholder.__name__ = _object.__name__ + Placeholder.__module__ = _object.__module__ + return _object if all(is_package_available(backend) for backend in applied_backends) else Placeholder + else: -def is_spacy_available() -> bool: - return importlib.util.find_spec("spacy") is not None + @functools.wraps(_object) + def wrapper(*args, **kwargs): + for backend in _object._backends: + raise_if_package_not_available(backend, object_name=_object.__name__) + return _object(*args, **kwargs) + return wrapper -NO_SPACY_ERROR_MSG = "You are trying to use some metrics requiring `spacy`, which is not available in your environment. Please install it using pip." + return inner_fn diff --git a/src/lighteval/utils/parallelism.py b/src/lighteval/utils/parallelism.py index 2e73f4c73..cedf1a07a 100644 --- a/src/lighteval/utils/parallelism.py +++ b/src/lighteval/utils/parallelism.py @@ -27,12 +27,7 @@ import torch -from lighteval.utils.imports import ( - NO_ACCELERATE_ERROR_MSG, - NO_NANOTRON_ERROR_MSG, - is_accelerate_available, - is_nanotron_available, -) +from lighteval.utils.imports import raise_if_package_not_available logger = logging.getLogger(__name__) @@ -126,21 +121,16 @@ def test_all_gather(accelerator=None, parallel_context=None): Args: accelerator (Optional): The accelerator object used for parallelism. parallel_context (Optional): The parallel context object used for parallelism. - - Raises: - ImportError: If the required accelerator or parallel context is not available. """ if accelerator: - if not is_accelerate_available(): - raise ImportError(NO_ACCELERATE_ERROR_MSG) + raise_if_package_not_available("accelerate") logger.info("Test gather tensor") test_tensor: torch.Tensor = torch.tensor([accelerator.process_index], device=accelerator.device) gathered_tensor: torch.Tensor = accelerator.gather(test_tensor) logger.info(f"gathered_tensor {gathered_tensor}, should be {list(range(accelerator.num_processes))}") accelerator.wait_for_everyone() elif parallel_context: - if not is_nanotron_available(): - raise ImportError(NO_NANOTRON_ERROR_MSG) + raise_if_package_not_available("nanotron") from nanotron import distributed as dist from nanotron import logging diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py new file mode 100644 index 000000000..694e5ec47 --- /dev/null +++ b/tests/test_dependencies.py @@ -0,0 +1,79 @@ +# MIT License +# +# Copyright (c) 2024 The HuggingFace Team +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import pytest + +from lighteval.utils.imports import Extra, is_package_available, requires + + +def test_requires(): + @requires("sglang") + class RandomModel: + pass + + assert RandomModel.__name__ == "RandomModel" + assert RandomModel.__class__.__name__ == "DummyObject" + + with pytest.raises( + ImportError, + match="Through the use of RandomModel, you requested the use of `sglang` for this evaluation, but it is not available in your current environment. Please install it using pip.", + ): + RandomModel() + + +def test_requires_with_extra(): + @requires(Extra.TGI) + class RandomModel: + pass + + with pytest.raises( + ImportError, + match=r"Through the use of RandomModel, you are trying to run an evaluation requiring tgi capabilities. Please install the required extra: `pip install lighteval\[tgi\]`", + ): + RandomModel() + + +def test_requires_with_wrong_dependency(): + with pytest.raises( + RuntimeError, + match="A dependency was specified with @requires, but it is not defined in the possible dependencies defined in the pyproject.toml: `random_dependency`", + ): + + @requires("random_dependency") + class RandomModel: + pass + + +def test_is_package_available(): + assert is_package_available("torch") + + +def test_is_package_unavailable(): + assert not is_package_available("tensorboardX") + + +def test_is_package_is_not_specified_in_pyproject_toml(): + with pytest.raises( + RuntimeError, + match="Package tensorflow was tested against, but isn't specified in the pyproject.toml file. Please specifyit as a potential dependency or an extra for it to be checked.", + ): + is_package_available("tensorflow") diff --git a/tests/unit/models/test_transformers_model.py b/tests/unit/models/test_transformers_model.py index 1cb2cf230..da7c925ae 100644 --- a/tests/unit/models/test_transformers_model.py +++ b/tests/unit/models/test_transformers_model.py @@ -398,9 +398,9 @@ def mock_gather(tensor): class TestTransformersModelUseChatTemplate(unittest.TestCase): @patch("lighteval.models.transformers.transformers_model.Accelerator") @patch("lighteval.models.transformers.transformers_model.TransformersModel._create_auto_model") - @patch("lighteval.utils.imports.is_accelerate_available") + @patch("lighteval.utils.imports.is_package_available") def test_transformers_model_use_chat_template_with_different_model_names( - self, mock_accelerator, mock_create_model, is_accelerate_available + self, mock_accelerator, mock_create_model, is_package_available ): """Test that TransformersModel correctly determines whether to use_chat_template or not automatically from the tokenizer config.""" test_cases = [ diff --git a/tests/unit/pipeline/test_reasoning_tags.py b/tests/unit/pipeline/test_reasoning_tags.py index 84dfb9e7e..f772970c4 100644 --- a/tests/unit/pipeline/test_reasoning_tags.py +++ b/tests/unit/pipeline/test_reasoning_tags.py @@ -35,7 +35,7 @@ from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig from lighteval.tasks.registry import Registry from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.imports import is_accelerate_available +from lighteval.utils.imports import is_package_available class TestPipelineReasoningTags(unittest.TestCase): @@ -129,7 +129,7 @@ def test_remove_reasoning_tags_enabled(self): ) # Initialize accelerator if available - if is_accelerate_available(): + if is_package_available("accelerate"): from accelerate import Accelerator Accelerator() @@ -175,7 +175,7 @@ def test_remove_reasoning_tags_enabled_tags_as_string(self): ) # Initialize accelerator if available - if is_accelerate_available(): + if is_package_available("accelerate"): from accelerate import Accelerator Accelerator() @@ -221,7 +221,7 @@ def test_remove_reasoning_tags_enabled_default_tags(self): ) # Initialize accelerator if available - if is_accelerate_available(): + if is_package_available("accelerate"): from accelerate import Accelerator Accelerator() @@ -264,7 +264,7 @@ def test_remove_reasoning_tags_disabled(self): ) # Initialize accelerator if available - if is_accelerate_available(): + if is_package_available("accelerate"): from accelerate import Accelerator Accelerator() @@ -310,7 +310,7 @@ def test_custom_reasoning_tags(self): ) # Initialize accelerator if available - if is_accelerate_available(): + if is_package_available("accelerate"): from accelerate import Accelerator Accelerator() @@ -356,7 +356,7 @@ def test_multiple_reasoning_tags(self): ) # Initialize accelerator if available - if is_accelerate_available(): + if is_package_available("accelerate"): from accelerate import Accelerator Accelerator() diff --git a/tests/unit/utils/test_caching.py b/tests/unit/utils/test_caching.py index 47ea599ae..7ab8644be 100644 --- a/tests/unit/utils/test_caching.py +++ b/tests/unit/utils/test_caching.py @@ -32,6 +32,7 @@ from lighteval.models.model_output import ModelResponse from lighteval.tasks.requests import Doc, SamplingMethod from lighteval.utils.cache_management import SampleCache +from lighteval.utils.imports import Extra, is_package_available class TestCaching(unittest.TestCase): @@ -177,12 +178,9 @@ def _test_cache(self, model: LightevalModel, test_cases): @patch("lighteval.models.transformers.transformers_model.TransformersModel._loglikelihood_tokens") @patch("lighteval.models.transformers.transformers_model.TransformersModel._padded_greedy_until") - @patch("lighteval.utils.imports.is_accelerate_available") @patch("lighteval.models.transformers.transformers_model.Accelerator") @patch("lighteval.models.transformers.transformers_model.TransformersModel._create_auto_model") - def test_cache_transformers( - self, mock_create_model, mock_accelerator, mock_is_accelerate_available, mock_greedy_until, mock_loglikelihood - ): + def test_cache_transformers(self, mock_create_model, mock_accelerator, mock_greedy_until, mock_loglikelihood): from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig # Skip the model creation phase @@ -192,7 +190,6 @@ def test_cache_transformers( mock_accelerator_instance = Mock() mock_accelerator_instance.device = torch.device("cpu") mock_accelerator.return_value = mock_accelerator_instance - mock_is_accelerate_available = False # noqa F841 mock_greedy_until.return_value = self.model_responses mock_loglikelihood.return_value = self.model_responses @@ -237,9 +234,8 @@ def test_cache_vllm(self, mock_create_model, mock_greedy_until, mock_loglikeliho @patch("lighteval.models.endpoints.tgi_model.ModelClient._loglikelihood") def test_cache_tgi(self, mock_loglikelihood, mock_greedy_until, mock_requests_get): from lighteval.models.endpoints.tgi_model import ModelClient, TGIModelConfig - from lighteval.utils.imports import is_tgi_available - if not is_tgi_available(): + if not is_package_available(Extra.TGI): pytest.skip("Skipping because missing the imports") # Mock TGI requests @@ -320,12 +316,9 @@ def test_cache_sglang( ) @patch("lighteval.models.transformers.vlm_transformers_model.VLMTransformersModel._greedy_until") - @patch("lighteval.utils.imports.is_accelerate_available") @patch("lighteval.models.transformers.vlm_transformers_model.Accelerator") @patch("lighteval.models.transformers.vlm_transformers_model.VLMTransformersModel._create_auto_model") - def test_cache_vlm_transformers( - self, mock_create_model, mock_accelerator, is_accelerate_available, mock_greedy_until - ): + def test_cache_vlm_transformers(self, mock_create_model, mock_accelerator, mock_greedy_until): from lighteval.models.transformers.vlm_transformers_model import ( VLMTransformersModel, VLMTransformersModelConfig, @@ -335,7 +328,6 @@ def test_cache_vlm_transformers( mock_accelerator_instance = Mock() mock_accelerator_instance.device = torch.device("cpu") mock_accelerator.return_value = mock_accelerator_instance - is_accelerate_available = False # noqa F841 # Skip the model creation phase mock_create_model = Mock() # noqa F841 diff --git a/tests/utils.py b/tests/utils.py index 7954b3531..3b68dd631 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -35,7 +35,7 @@ from lighteval.tasks.registry import Registry from lighteval.tasks.requests import Doc from lighteval.utils.cache_management import SampleCache -from lighteval.utils.imports import is_accelerate_available +from lighteval.utils.imports import is_package_available class FakeModelConfig(ModelConfig): @@ -117,7 +117,7 @@ def load_tasks(self): # This is due to logger complaining we have no initialised the accelerator # It's hard to mock as it's global singleton - if is_accelerate_available(): + if is_package_available("accelerate"): from accelerate import Accelerator Accelerator()