diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ce0a74ac..541d2e980 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ # Changelog -## [v0.3.0] - 2025-06-10 +## [v0.3.1] - 2025-06-10 + +### Added +- New pytest marker `dry_runnable` for tests that can run without inference. +- Enhanced `make` targets with dry-run capabilities for improved test coverage: + - `make test-xdist` (or `make t`): Runs all non-inference tests **plus inference tests** that support dry-runs - fast and resource-efficient + - `make test-inference` (or `make ti`): Runs tests requiring actual inference, with actual inference (slow and costly) +- Parallel test execution using `pytest-xdist` (`-n auto`) enabled for: + - GitHub Actions workflows + - Codex test targets + +### Changed +- Domain validation is now less restrictive in pipeline TOML: the `definition` attribute is now `Optional` + +## [v0.3.0] - 2025-06-09 ### Highlights diff --git a/Makefile b/Makefile index fc6c08af4..4366989f3 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,8 @@ VENV_PIPELEX := $(VIRTUAL_ENV)/bin/pipelex UV_MIN_VERSION = $(shell grep -m1 'required-version' pyproject.toml | sed -E 's/.*= *"([^<>=, ]+).*/\1/') +USUAL_PYTEST_MARKERS := "(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)" + define PRINT_TITLE $(eval PROJECT_PART := [$(PROJECT_NAME)]) $(eval TARGET_PART := ($@)) @@ -203,17 +205,17 @@ cleanall: cleanderived cleanenv cleanlibraries codex-tests: env $(call PRINT_TITLE,"Unit testing for Codex") @echo "• Running unit tests for Codex (excluding inference and codex_disabled)" - $(VENV_PYTEST) --exitfirst --quiet -m "not (inference or codex_disabled or pipelex_api)" || [ $$? = 5 ] + $(VENV_PYTEST) -n auto --exitfirst --quiet -m "(dry_runnable or not inference) and not (needs_output or pipelex_api)" || [ $$? = 5 ] gha-tests: env $(call PRINT_TITLE,"Unit testing for github actions") @echo "• Running unit tests for github actions (excluding inference and gha_disabled)" - $(VENV_PYTEST) --exitfirst --quiet -m "not (inference or gha_disabled or pipelex_api)" || [ $$? = 5 ] + $(VENV_PYTEST) -n auto --exitfirst --quiet -m "(dry_runnable or not inference) and not (gha_disabled or pipelex_api)" || [ $$? = 5 ] run-all-tests: env $(call PRINT_TITLE,"Running all unit tests") @echo "• Running all unit tests" - $(VENV_PYTEST) --exitfirst --quiet + $(VENV_PYTEST) -n auto --exitfirst --quiet run-manual-trigger-gha-tests: env $(call PRINT_TITLE,"Running GHA tests") @@ -229,18 +231,18 @@ test: env $(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above") @echo "• Running unit tests" @if [ -n "$(TEST)" ]; then \ - $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ else \ - $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ fi test-xdist: env $(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above") @echo "• Running unit tests" @if [ -n "$(TEST)" ]; then \ - $(VENV_PYTEST) -n auto -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) -n auto -m $(USUAL_PYTEST_MARKERS) -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ else \ - $(VENV_PYTEST) -n auto -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) -n auto -m $(USUAL_PYTEST_MARKERS) -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ fi t: test-xdist @@ -250,9 +252,9 @@ test-quiet: env $(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above") @echo "• Running unit tests" @if [ -n "$(TEST)" ]; then \ - $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) -m $(USUAL_PYTEST_MARKERS) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ else \ - $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) -m $(USUAL_PYTEST_MARKERS) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ fi tq: test-quiet @@ -262,9 +264,9 @@ test-with-prints: env $(call PRINT_TITLE,"Unit testing with prints and our rich logs") @echo "• Running unit tests" @if [ -n "$(TEST)" ]; then \ - $(VENV_PYTEST) -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ else \ - $(VENV_PYTEST) -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ fi tp: test-with-prints @@ -273,9 +275,9 @@ tp: test-with-prints test-inference: env $(call PRINT_TITLE,"Unit testing") @if [ -n "$(TEST)" ]; then \ - $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ else \ - $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ fi ti: test-inference @@ -284,9 +286,9 @@ ti: test-inference test-ocr: env $(call PRINT_TITLE,"Unit testing ocr") @if [ -n "$(TEST)" ]; then \ - $(VENV_PYTEST) --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ else \ - $(VENV_PYTEST) --exitfirst -m "ocr" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "ocr" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ fi to: test-ocr @@ -295,9 +297,9 @@ to: test-ocr test-imgg: env $(call PRINT_TITLE,"Unit testing") @if [ -n "$(TEST)" ]; then \ - $(VENV_PYTEST) --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ else \ - $(VENV_PYTEST) --exitfirst -m "imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ + $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \ fi tg: test-imgg diff --git a/pipelex/core/domain.py b/pipelex/core/domain.py index 3d2081c3c..04cfbd35d 100644 --- a/pipelex/core/domain.py +++ b/pipelex/core/domain.py @@ -13,7 +13,7 @@ class SpecialDomain(StrEnum): class Domain(BaseModel): code: str - definition: str + definition: Optional[str] = None system_prompt: Optional[str] = None system_prompt_to_structure: Optional[str] = None prompt_template_to_structure: Optional[str] = None @@ -24,4 +24,4 @@ def __str__(self): @classmethod def make_default(cls) -> Self: - return cls(code=SpecialDomain.NATIVE, definition="") + return cls(code=SpecialDomain.NATIVE) diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py index 1871e70ff..f527f9f75 100644 --- a/pipelex/libraries/library_manager.py +++ b/pipelex/libraries/library_manager.py @@ -128,7 +128,10 @@ def _load_combo_libraries(self, library_paths: List[str]): library_name = toml_path.stem domain_code = library_dict.get("domain") if domain_code is None: - raise LibraryParsingError(f"Error loafing library '{library_name}' which has no domain set at '{toml_path}'") + raise LibraryParsingError( + f"Error loading library '{library_name}' which has no domain set at '{toml_path}'. " + "Just write 'domain = \"my_domain\"' at the top of the file." + ) domain_definition = library_dict.get("definition") if domain_definition is None: # we skip the domain without definition, it must be defined one and only one time in the domain library @@ -177,10 +180,6 @@ def _load_combo_libraries(self, library_paths: List[str]): def _load_library_dict(self, library_name: str, library_dict: Dict[str, Any], component_type: LibraryComponent): if domain_code := library_dict.pop("domain", None): - if not self.domain_library.get_domain(domain_code=domain_code): - raise LibraryParsingError( - f"Domain '{domain_code}' is has not been defined in the domain libraryn make sure it has exactlyone definition" - ) # domain is set at the root of the library self._load_library_components_from_recursive_dict( domain_code=domain_code, diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml index d7a47ff48..79431a752 100644 --- a/pipelex/pipelex.toml +++ b/pipelex/pipelex.toml @@ -244,4 +244,4 @@ pipe_stack_limit = 20 [pipelex.dry_run_config] apply_to_jinja2_rendering = false -text_gen_truncate_length = 256 \ No newline at end of file +text_gen_truncate_length = 256 diff --git a/pyproject.toml b/pyproject.toml index 252d88222..1130c1845 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pipelex" -version = "0.3.0" +version = "0.3.1" description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines." authors = [{ name = "Evotis S.A.S.", email = "evotis@pipelex.com" }] maintainers = [{ name = "Pipelex staff", email = "oss@pipelex.com" }] @@ -208,6 +208,7 @@ markers = [ "ocr: slow and costly due to ocr inference calls", "gha_disabled: tests that should not run in GitHub Actions", "codex_disabled: tests that should not run in Codex", + "dry_runnable: tests that can be run in dry-run mode", ] minversion = "8.0" diff --git a/tests/pipelex/pipelex_asynch/test_pipe_batch.py b/tests/pipelex/pipelex_asynch/test_pipe_batch.py index d2b9eb4a3..be772c723 100644 --- a/tests/pipelex/pipelex_asynch/test_pipe_batch.py +++ b/tests/pipelex/pipelex_asynch/test_pipe_batch.py @@ -10,6 +10,7 @@ from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_report_delegate +@pytest.mark.dry_runnable @pytest.mark.llm @pytest.mark.inference @pytest.mark.asyncio(loop_scope="class") diff --git a/tests/pipelex/pipelex_asynch/test_pipe_imgg.py b/tests/pipelex/pipelex_asynch/test_pipe_imgg.py index ad501bcfb..3ef3e5f71 100644 --- a/tests/pipelex/pipelex_asynch/test_pipe_imgg.py +++ b/tests/pipelex/pipelex_asynch/test_pipe_imgg.py @@ -11,6 +11,7 @@ from tests.pipelex.test_data import IMGGTestCases +@pytest.mark.dry_runnable @pytest.mark.imgg @pytest.mark.inference @pytest.mark.asyncio(loop_scope="class") diff --git a/tests/pipelex/pipelex_asynch/test_pipe_llm.py b/tests/pipelex/pipelex_asynch/test_pipe_llm.py index 6b895a69d..86535a7ca 100644 --- a/tests/pipelex/pipelex_asynch/test_pipe_llm.py +++ b/tests/pipelex/pipelex_asynch/test_pipe_llm.py @@ -16,6 +16,7 @@ from tests.pipelex.test_data import PipeTestCases +@pytest.mark.dry_runnable @pytest.mark.llm @pytest.mark.inference @pytest.mark.asyncio(loop_scope="class") diff --git a/tests/pipelex/pipelex_asynch/test_pipe_ocr.py b/tests/pipelex/pipelex_asynch/test_pipe_ocr.py index 9ebe82096..349da9321 100644 --- a/tests/pipelex/pipelex_asynch/test_pipe_ocr.py +++ b/tests/pipelex/pipelex_asynch/test_pipe_ocr.py @@ -13,6 +13,7 @@ from tests.pipelex.test_data import PipeOcrTestCases +@pytest.mark.dry_runnable @pytest.mark.ocr @pytest.mark.inference @pytest.mark.asyncio(loop_scope="class") diff --git a/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py b/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py index ba7ab0c20..023a733e7 100644 --- a/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py +++ b/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py @@ -17,6 +17,7 @@ from tests.pipelex.test_data import PipeTestCases +@pytest.mark.dry_runnable @pytest.mark.llm @pytest.mark.ocr @pytest.mark.inference diff --git a/tests/test_pipelines/failure_modes.toml b/tests/test_pipelines/failure_modes.toml index 6abb942ea..a46283c99 100644 --- a/tests/test_pipelines/failure_modes.toml +++ b/tests/test_pipelines/failure_modes.toml @@ -1,7 +1,6 @@ domain = "failure_modes" -definition = "This domain is for testing failure modes" [concept]