From c2805f6a6781a1a86de9aeebf50bd59788295080 Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Wed, 26 Jun 2024 11:31:19 +0300 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20Small=20scripts=20a?= =?UTF-8?q?nd=20configurations=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * `run` & `fix` commands are added to the Makefile * `pyproject.toml` now includes more specifications for code quality tools --- Makefile | 14 ++++++++++++-- pyproject.toml | 30 +++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 53f35376..71ca207c 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +run: + python -m src.guidellm.main + install: pip install -r requirements.txt @@ -13,7 +16,7 @@ quality: style: ruff format src tests isort src tests - flake8 src tests --max-line-length 88 + flake8 src tests # test: # pytest tests @@ -31,4 +34,11 @@ clean: rm -rf .mypy_cache rm -rf .pytest_cache -.PHONY: install install-dev quality style test test-unit test-integration test-e2e test-smoke test-sanity test-regression build clean + +fix: + python -m black src tests + python -m isort src tests + python -m ruff format src tests + + +.PHONY: run install install-dev quality style test test-unit test-integration test-e2e test-smoke test-sanity test-regression build clean fix diff --git a/pyproject.toml b/pyproject.toml index 71b17670..064f496a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,9 +8,32 @@ target-version = ['py38'] [tool.isort] profile = "black" +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +line_length = 88 +src_paths = ["."] [tool.mypy] -files = "src/guidellm" +python_version = '3.8' +files = ['*.py'] +exclude = ["venv", "docs"] + +show_error_codes = true +namespace_packages = false +check_untyped_defs = true + +warn_redundant_casts = true +warn_unused_ignores = true + +# Silint "type import errors" as our 3rd-party libs does not have types +# Check: https://mypy.readthedocs.io/en/latest/config_file.html#import-discovery +follow_imports = 'silent' + +[[tool.mypy.overrides]] +module = [] +ignore_missing_imports=true [tool.ruff] exclude = ["build", "dist", "env", ".venv"] @@ -20,6 +43,11 @@ lint.select = ["E", "F", "W"] max-line-length = 88 [tool.pytest.ini_options] +addopts = '-s -vvv --cache-clear' +asyncio_mode = 'auto' +cache_dir = '/tmp' +python_files = 'tests.py test_*.py' +python_functions = 'test_* *_test' markers = [ "smoke: quick tests to check basic functionality", "sanity: detailed tests to ensure major functions work correctly", From 54c8e143b89bd787d15ea20ad14db96801ade50e Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Wed, 26 Jun 2024 11:41:34 +0300 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=90=9B=20`create=5Fbackend`=20invalid?= =?UTF-8?q?=20condition=20is=20fixed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * `create_backend` is no longer fail due to inproper condition * rename: `BackendTypes` -> `BackendType` --- src/guidellm/backend/__init__.py | 4 ++-- src/guidellm/backend/base.py | 18 ++++++++++-------- src/guidellm/backend/openai.py | 4 ++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/guidellm/backend/__init__.py b/src/guidellm/backend/__init__.py index cc5c740e..f9d5541f 100644 --- a/src/guidellm/backend/__init__.py +++ b/src/guidellm/backend/__init__.py @@ -1,9 +1,9 @@ -from .base import Backend, BackendTypes, GenerativeResponse +from .base import Backend, BackendType, GenerativeResponse from .openai import OpenAIBackend __all__ = [ "Backend", - "BackendTypes", + "BackendType", "GenerativeResponse", "OpenAIBackend", ] diff --git a/src/guidellm/backend/base.py b/src/guidellm/backend/base.py index 22aab806..b09ce376 100644 --- a/src/guidellm/backend/base.py +++ b/src/guidellm/backend/base.py @@ -2,17 +2,17 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum -from typing import Iterator, List, Optional, Type, Union +from typing import Iterator, List, Optional, Type from loguru import logger from guidellm.core.request import TextGenerationRequest from guidellm.core.result import TextGenerationResult -__all__ = ["Backend", "BackendTypes", "GenerativeResponse"] +__all__ = ["Backend", "BackendType", "GenerativeResponse"] -class BackendTypes(Enum): +class BackendType(str, Enum): TEST = "test" OPENAI_SERVER = "openai_server" @@ -39,12 +39,12 @@ class Backend(ABC): _registry = {} @staticmethod - def register_backend(backend_type: BackendTypes): + def register_backend(backend_type: BackendType): """ A decorator to register a backend class in the backend registry. :param backend_type: The type of backend to register. - :type backend_type: BackendTypes + :type backend_type: BackendType """ def inner_wrapper(wrapped_class: Type["Backend"]): @@ -54,21 +54,23 @@ def inner_wrapper(wrapped_class: Type["Backend"]): return inner_wrapper @staticmethod - def create_backend(backend_type: Union[str, BackendTypes], **kwargs) -> "Backend": + def create_backend(backend_type: BackendType, **kwargs) -> "Backend": """ Factory method to create a backend based on the backend type. :param backend_type: The type of backend to create. - :type backend_type: BackendTypes + :type backend_type: BackendType :param kwargs: Additional arguments for backend initialization. :type kwargs: dict :return: An instance of a subclass of Backend. :rtype: Backend """ logger.info(f"Creating backend of type {backend_type}") - if backend_type not in Backend._registry: + + if backend_type not in Backend._registry.keys(): logger.error(f"Unsupported backend type: {backend_type}") raise ValueError(f"Unsupported backend type: {backend_type}") + return Backend._registry[backend_type](**kwargs) def submit(self, request: TextGenerationRequest) -> TextGenerationResult: diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index ce9f6c2d..d2656be8 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -4,13 +4,13 @@ from loguru import logger from transformers import AutoTokenizer -from guidellm.backend import Backend, BackendTypes, GenerativeResponse +from guidellm.backend import Backend, BackendType, GenerativeResponse from guidellm.core.request import TextGenerationRequest __all__ = ["OpenAIBackend"] -@Backend.register_backend(BackendTypes.OPENAI_SERVER) +@Backend.register_backend(BackendType.OPENAI_SERVER) class OpenAIBackend(Backend): """ An OpenAI backend implementation for the generative AI result. From 0bec1fd49db0d04b66c4172170db63b58b7b432c Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Thu, 27 Jun 2024 13:06:10 +0300 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20`RequestGenerator`?= =?UTF-8?q?=20implements=20`AsyncIterator`=20protocol=20Unit=20tests=20are?= =?UTF-8?q?=20fixed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/guidellm/request/base.py | 84 ++++++++++++++++----------------- tests/unit/request/test_base.py | 28 +++++------ 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/src/guidellm/request/base.py b/src/guidellm/request/base.py index e5b20034..2612025a 100644 --- a/src/guidellm/request/base.py +++ b/src/guidellm/request/base.py @@ -1,6 +1,6 @@ import asyncio from abc import ABC, abstractmethod -from typing import Iterator, Optional, Union +from typing import AsyncIterator, Generator, Iterator, Optional, Union from loguru import logger from transformers import AutoTokenizer, PreTrainedTokenizer @@ -25,14 +25,15 @@ class RequestGenerator(ABC): def __init__( self, + *_, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, mode: str = "async", async_queue_size: int = 50, ): - self._async_queue_size = async_queue_size - self._mode = mode - self._queue = asyncio.Queue(maxsize=async_queue_size) - self._stop_event = asyncio.Event() + self._async_queue_size: int = async_queue_size + self._mode: str = mode + self._queue: asyncio.Queue = asyncio.Queue(maxsize=async_queue_size) + self._stop_event: asyncio.Event = asyncio.Event() if tokenizer is not None: self._tokenizer = ( @@ -59,24 +60,38 @@ def __repr__(self) -> str: f"tokenizer={self._tokenizer})" ) - def __iter__(self) -> Iterator[TextGenerationRequest]: + def __iter__(self) -> Generator[TextGenerationRequest, None, None]: """ Provide an iterator interface to generate new requests. + """ - :return: An iterator over result requests. - :rtype: Iterator[TextGenerationRequest] - """ - if self.mode == "async": - while not self._stop_event.is_set(): - try: - item = self._queue.get_nowait() - self._queue.task_done() - yield item - except asyncio.QueueEmpty: - continue - else: - while not self._stop_event.is_set(): - yield self.create_item() + while not self._stop_event.is_set(): + yield self.create_item() + + def __aiter__(self) -> AsyncIterator["TextGenerationRequest"]: + """ + Provide an async iterator interface to generate new requests. + """ + + return self + + async def __anext__(self) -> "TextGenerationRequest": + """ + Asynchronously get the next item from the queue or wait if the queue is empty. + """ + + asyncio.create_task(self._populate_queue()) + + while not self._stop_event.is_set(): + try: + item = self._queue.get_nowait() + self._queue.task_done() + return item + except asyncio.QueueEmpty: + # Throttle and release the control + await asyncio.sleep(0) + + raise StopAsyncIteration @property def tokenizer(self) -> Optional[PreTrainedTokenizer]: @@ -118,31 +133,13 @@ def create_item(self) -> TextGenerationRequest: """ raise NotImplementedError() - def start(self): - """ - Start the background task that populates the queue. - """ - if self.mode == "async": - try: - loop = asyncio.get_running_loop() - logger.info("Using existing event loop") - except RuntimeError: - raise RuntimeError("No running event loop found for async mode") - - loop.call_soon_threadsafe( - lambda: asyncio.create_task(self._populate_queue()) - ) - logger.info( - f"RequestGenerator started in async mode with queue size: " - f"{self._async_queue_size}" - ) - else: - logger.info("RequestGenerator started in sync mode") - def stop(self): """ Stop the background task that populates the queue. """ + + # TODO: Consider moving to the __anext__ + logger.info("Stopping RequestGenerator...") self._stop_event.set() logger.info("RequestGenerator stopped") @@ -151,13 +148,16 @@ async def _populate_queue(self): """ Populate the request queue in the background. """ + while not self._stop_event.is_set(): if self._queue.qsize() < self._async_queue_size: item = self.create_item() await self._queue.put(item) + logger.debug( f"Item added to queue. Current queue size: {self._queue.qsize()}" ) else: - await asyncio.sleep(0.1) + await asyncio.sleep(0) + logger.info("RequestGenerator stopped populating queue") diff --git a/tests/unit/request/test_base.py b/tests/unit/request/test_base.py index 31623baa..5c420ef9 100644 --- a/tests/unit/request/test_base.py +++ b/tests/unit/request/test_base.py @@ -1,3 +1,6 @@ +import asyncio +from collections.abc import AsyncGenerator + import pytest from guidellm.core.request import TextGenerationRequest @@ -22,33 +25,30 @@ def test_request_generator_sync(): if len(items) == 5: break - assert len(items) == 5 assert items[0].prompt == "Test prompt" @pytest.mark.smoke -@pytest.mark.asyncio -def test_request_generator_async(): +async def test_request_generator_async(): generator = TestRequestGenerator(mode="async", async_queue_size=10) + assert generator.mode == "async" assert generator.async_queue_size == 10 assert generator.tokenizer is None - generator.start() - items = [] - for item in generator: - items.append(item) + try: + async for item in generator: + items.append(item) - if len(items) == 5: - break + if len(items) == 5: + break + finally: + generator.stop() - generator.stop() assert generator._stop_event.is_set() - - assert len(items) == 5 - assert items[0].prompt == "Test prompt" - assert items[-1].prompt == "Test prompt" + for item in items: + assert item.prompt == "Test prompt" @pytest.mark.regression From 6f1443c1428b2754a697f8a72d039d4b75acd865 Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Thu, 27 Jun 2024 13:07:59 +0300 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=97=91=EF=B8=8F=20Unused=20aliases=20?= =?UTF-8?q?are=20removed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 71ca207c..c79986f1 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,27 @@ -run: - python -m src.guidellm.main - install: pip install -r requirements.txt + install-dev: pip install -e .[dev] + quality: ruff check src tests isort --check-only src tests flake8 src tests --max-line-length 88 - mypy src + style: - ruff format src tests - isort src tests - flake8 src tests + python -m ruff format src tests + python -m isort src tests + python -m flake8 src tests -# test: -# pytest tests build: python setup.py sdist bdist_wheel + clean: rm -rf __pycache__ rm -rf build @@ -35,10 +33,4 @@ clean: rm -rf .pytest_cache -fix: - python -m black src tests - python -m isort src tests - python -m ruff format src tests - - -.PHONY: run install install-dev quality style test test-unit test-integration test-e2e test-smoke test-sanity test-regression build clean fix +.PHONY: install install-dev quality style build clean From f1ee042f8171f890e73175728e62bd6338043f88 Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Thu, 27 Jun 2024 13:09:54 +0300 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/guidellm/core/result.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/guidellm/core/result.py b/src/guidellm/core/result.py index a72a03bc..a291ed10 100644 --- a/src/guidellm/core/result.py +++ b/src/guidellm/core/result.py @@ -599,3 +599,6 @@ def add_benchmark(self, benchmark: TextGenerationBenchmark): """ self._benchmarks.append(benchmark) logger.debug(f"Added result: {benchmark}") + + def to_dict(self) -> dict: + raise NotImplementedError From 8eb807b2e2181c7e36525784f094028353328541 Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Thu, 27 Jun 2024 13:10:21 +0300 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=9A=A7=20Issue=20iwth=20OpenAI=20vers?= =?UTF-8?q?ions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/guidellm/backend/openai.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index d2656be8..689225e2 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -1,7 +1,7 @@ from typing import Any, Iterator, List, Optional -import openai from loguru import logger +from openai import OpenAI from transformers import AutoTokenizer from guidellm.backend import Backend, BackendType, GenerativeResponse @@ -53,8 +53,7 @@ def __init__( path_incl = path if path else "" self.target = f"http://{host}{port_incl}{path_incl}" - openai.api_base = self.target - openai.api_key = api_key + self._openai_client = OpenAI(api_key=api_key, base_url=self.target) if not model: self.model = self.default_model() @@ -88,7 +87,7 @@ def make_request( if self.request_args: request_args.update(self.request_args) - response = openai.Completion.create( + response = self._openai_client.completions.create( engine=self.model, prompt=request.prompt, stream=True, @@ -129,7 +128,18 @@ def available_models(self) -> List[str]: :return: A list of available models. :rtype: List[str] """ - models = [model["id"] for model in openai.Engine.list()["data"]] + + # FIX:You tried to access openai.Engine, but this is no longer supported + # in openai>=1.0.0 - see the README at + # https://github.com/openai/openai-python for the API. + # You can run `openai migrate` to automatically upgrade your codebase + # to use the 1.0.0 interface. Alternatively, you can pin your installation + # to the old version, e.g. `pip install openai==0.28` + # A detailed migration guide is available here: + # https://github.com/openai/openai-python/discussions/742 + + # in progress + models = [model["id"] for model in self._openai_client.models.list()["data"]] logger.info(f"Available models: {models}") return models From 306d0672130ee08f10396423e90ec10acdc5702d Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Thu, 27 Jun 2024 16:03:56 +0300 Subject: [PATCH 7/7] =?UTF-8?q?=E2=9E=95=20`pytest-asyncio`=20is=20added?= =?UTF-8?q?=20to=20dev=20dependencies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9075d84c..f0fe8bea 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ def _setup_long_description() -> Tuple[str, str]: extras_require={ 'dev': [ 'pytest', + 'pytest-asyncio', 'sphinx', 'ruff', 'mypy',