diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..a648d50ca --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,98 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PYTHONDONTWRITEBYTECODE: "1" + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2 + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.2.0 + with: + python-version: "3.11" + - name: Install Ruff + run: python -m pip install ruff==0.11.4 + - name: Run high-confidence lint checks + run: ruff check src tests setup.py --select E4,E7,E9,F --ignore E731 + + unit-tests: + name: CPU unit tests (Python 3.9) + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + WANDB_MODE: disabled + steps: + - name: Check out repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2 + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.2.0 + with: + python-version: "3.9" + cache: pip + cache-dependency-path: | + requirements.txt + setup.py + - name: Install LMFlow and test dependencies + run: python -m pip install -e ".[develop]" + - name: Run offline CPU tests + run: >- + python -m pytest -q --strict-markers + -m "not gpu and not slow and not online and not optional_backend" + + package: + name: Build package + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2 + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.2.0 + with: + python-version: "3.11" + cache: pip + - name: Install build tools + run: python -m pip install build twine + - name: Build distributions + run: python -m build + - name: Validate distributions + run: python -m twine check dist/* + - name: Install wheel without runtime dependencies + run: python -m pip install --no-deps --force-reinstall dist/*.whl + - name: Verify installed package metadata + run: | + python - <<'PY' + from importlib.metadata import metadata, version + + package_metadata = metadata("lmflow") + assert version("lmflow") + assert package_metadata["Requires-Python"] == ">=3.9" + PY + - name: Upload distributions + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: python-distributions + path: dist/ + if-no-files-found: error diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index 36ae8d0b9..1ea066b15 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -1,24 +1,59 @@ name: Docs -on: [push, pull_request, workflow_dispatch] + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: pages-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - docs: + build: + name: Build documentation runs-on: ubuntu-latest + timeout-minutes: 20 steps: - - uses: actions/checkout@v3 - - name: python environment setup - uses: actions/setup-python@v5.1.0 - with: + - name: Check out repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2 + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.2.0 + with: python-version: "3.11" + cache: pip + cache-dependency-path: docs/requirements.txt - name: Install dependencies - run: | - pip install -r ./docs/requirements.txt - - name: Sphinx build - run: | - sphinx-build docs/source _build - - name: Deploy - uses: peaceiris/actions-gh-pages@v3 + run: python -m pip install -r docs/requirements.txt + - name: Build documentation + run: sphinx-build -b html docs/source _build/html + - name: Configure GitHub Pages + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + - name: Upload Pages artifact + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 with: - publish_branch: gh-pages - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: _build/ - force_orphan: true + path: _build/html + + deploy: + name: Deploy documentation + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/pyproject.toml b/pyproject.toml index 96f807b1c..ed387b735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,19 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "tests/**/*.py" = ["F841"] +"tests/pipeline/test_sglang_infernecer.py" = ["E402"] +"tests/pipeline/test_vllm_inferencer.py" = ["E402"] "src/lmflow/utils/conversation_template/*.py" = ["E501"] "src/lmflow/utils/llava_conversation_lib.py" = ["E501"] "src/lmflow/utils/constants.py" = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "gpu: requires a GPU", + "slow: takes substantially longer than the unit-test suite", + "online: requires network access or a remote model/dataset", + "optional_backend: requires an optional inference backend", + "lmflow_core: tests for core LMFlow functionality", + "dothis: mark for developers to run explicitly", +] diff --git a/scripts/run_unittest.sh b/scripts/run_unittest.sh index 5936b7999..464e11502 100755 --- a/scripts/run_unittest.sh +++ b/scripts/run_unittest.sh @@ -1,3 +1,4 @@ #!/bin/bash -python -m unittest discover +python -m pytest -q --strict-markers \ + -m "not gpu and not slow and not online and not optional_backend" diff --git a/setup.py b/setup.py index ec990542f..ffd11ad22 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ], - requires_python=">=3.9", + python_requires=">=3.9", ) # optionals diff --git a/src/lmflow/pipeline/inferencer.py b/src/lmflow/pipeline/inferencer.py index ec64b290d..7d9422330 100644 --- a/src/lmflow/pipeline/inferencer.py +++ b/src/lmflow/pipeline/inferencer.py @@ -5,6 +5,7 @@ import logging import os import subprocess +import sys import numpy as np import torch @@ -626,7 +627,7 @@ def inference( def code_exec(self, code): # Execute the code - result = subprocess.run(["python", "-c", code], capture_output=True, text=True) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) # Print the result if result.returncode == 0: diff --git a/src/lmflow/pipeline/sglang_inferencer.py b/src/lmflow/pipeline/sglang_inferencer.py index f9d012939..1eedd9aae 100644 --- a/src/lmflow/pipeline/sglang_inferencer.py +++ b/src/lmflow/pipeline/sglang_inferencer.py @@ -1,9 +1,8 @@ #!/usr/bin/env python # Copyright 2024 Statistics and Machine Learning Research Group. All rights reserved. -import json import logging import os -from typing import Optional, Union +from typing import Optional from transformers import AutoTokenizer diff --git a/tests/datasets/test_dataset.py b/tests/datasets/test_dataset.py index c3c9fa358..18a7cb4ab 100644 --- a/tests/datasets/test_dataset.py +++ b/tests/datasets/test_dataset.py @@ -14,8 +14,8 @@ import json import os +import tempfile import unittest -from pathlib import Path from lmflow.args import DatasetArguments from lmflow.datasets.dataset import Dataset @@ -23,15 +23,24 @@ class DatasetTest(unittest.TestCase): def test_init(self): - dataset_dir = "data/example_dataset/train" - data_args = DatasetArguments(dataset_path=dataset_dir) - dataset = Dataset(data_args, backend="huggingface") - hf_dataset = dataset.get_backend_dataset() - - with open(os.path.join(Path(dataset_dir), "train_50.json")) as fin: - json_obj = json.load(fin) - for i in range(len(hf_dataset)): - self.assertEqual(json_obj["instances"][i], hf_dataset[i]) + json_obj = { + "type": "text2text", + "instances": [ + {"input": "INPUT 1", "output": "OUTPUT 1"}, + {"input": "INPUT 2", "output": "OUTPUT 2"}, + ], + } + with tempfile.TemporaryDirectory() as dataset_dir: + with open(os.path.join(dataset_dir, "train.json"), "w", encoding="utf-8") as fout: + json.dump(json_obj, fout) + + data_args = DatasetArguments(dataset_path=dataset_dir) + dataset = Dataset(data_args, backend="huggingface") + hf_dataset = dataset.get_backend_dataset() + + self.assertEqual(len(hf_dataset), len(json_obj["instances"])) + for expected, actual in zip(json_obj["instances"], hf_dataset): + self.assertEqual(expected, actual) def test_create_from_dict(self): data_dict = { diff --git a/tests/models/test_hf_decoder_model.py b/tests/models/test_hf_decoder_model.py index 4a07c32ff..6ef239274 100644 --- a/tests/models/test_hf_decoder_model.py +++ b/tests/models/test_hf_decoder_model.py @@ -19,6 +19,8 @@ import os from pathlib import Path +import pytest + from lmflow.args import DatasetArguments, ModelArguments from lmflow.datasets.dataset import Dataset from lmflow.models.hf_decoder_model import HFDecoderModel @@ -28,6 +30,8 @@ ) from lmflow.utils.conversation_template import PRESET_TEMPLATES +pytestmark = pytest.mark.online + SAMPLE_TEXT = "Defintion: In this task, we ask you to write an answer to a question that involves events that may be stationary (not changing over time) or transient (changing over time). For example, the sentence \"he was born in the U.S.\" contains a stationary event since it will last forever; however, \"he is hungry\" contains a transient event since it will remain true for a short period of time. Note that a lot of the questions could have more than one correct answer. We only need a single most-likely answer. Please try to keep your \"answer\" as simple as possible. Concise and simple \"answer\" is preferred over those complex and verbose ones. \\n Input: Sentence: It's hail crackled across the comm, and Tara spun to retake her seat at the helm. \nQuestion: Will the hail storm ever end? \\n Output: NA \\n\\n" @@ -671,4 +675,4 @@ def test_inference(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/models/test_tool_inferencer.py b/tests/models/test_tool_inferencer.py index 8d66135a4..2f5744188 100644 --- a/tests/models/test_tool_inferencer.py +++ b/tests/models/test_tool_inferencer.py @@ -1,6 +1,5 @@ import unittest -from lmflow.args import DatasetArguments, InferencerArguments, ModelArguments from lmflow.pipeline.inferencer import ToolInferencer CODE_1 = 'print("hello world")' @@ -13,11 +12,10 @@ class ToolInferencerTest(unittest.TestCase): - def set_up(self): - model_args = ModelArguments(model_name_or_path="codellama/CodeLlama-7b-instruct-hf") - inferencer_args = InferencerArguments() - data_args = DatasetArguments() - self.toolinf = ToolInferencer(model_args, data_args, inferencer_args) + def setUp(self): + # code_exec does not use model state; bypass model initialization so this + # remains a fast, offline unit test. + self.toolinf = object.__new__(ToolInferencer) def test_code_exec_1(self, code=CODE_1, expected_output=RES_1): toolinf_res = self.toolinf.code_exec(code) diff --git a/tests/pipeline/test_auto_pipeline.py b/tests/pipeline/test_auto_pipeline.py index 621c2d920..d40af0562 100644 --- a/tests/pipeline/test_auto_pipeline.py +++ b/tests/pipeline/test_auto_pipeline.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch from lmflow.args import DatasetArguments, EvaluatorArguments, FinetunerArguments, InferencerArguments, ModelArguments from lmflow.pipeline.auto_pipeline import AutoPipeline @@ -14,7 +15,8 @@ def test_get_evaluator_pipeline(self): model_args = ModelArguments(model_name_or_path=MODEL_NAME) dataset_args = DatasetArguments() evaluator_args = EvaluatorArguments() - pipeline = AutoPipeline.get_pipeline("evaluator", model_args, dataset_args, evaluator_args) + with patch.object(Evaluator, "__init__", return_value=None): + pipeline = AutoPipeline.get_pipeline("evaluator", model_args, dataset_args, evaluator_args) self.assertTrue(isinstance(pipeline, Evaluator)) @@ -22,7 +24,8 @@ def test_get_finetuner_pipeline(self): model_args = ModelArguments(model_name_or_path=MODEL_NAME) dataset_args = DatasetArguments() finetuner_args = FinetunerArguments(output_dir="~/tmp") - pipeline = AutoPipeline.get_pipeline("finetuner", model_args, dataset_args, finetuner_args) + with patch.object(Finetuner, "__init__", return_value=None): + pipeline = AutoPipeline.get_pipeline("finetuner", model_args, dataset_args, finetuner_args) self.assertTrue(isinstance(pipeline, Finetuner)) @@ -30,7 +33,8 @@ def test_get_inferencer_pipeline(self): model_args = ModelArguments(model_name_or_path=MODEL_NAME) dataset_args = DatasetArguments() inferencer_args = InferencerArguments() - pipeline = AutoPipeline.get_pipeline("inferencer", model_args, dataset_args, inferencer_args) + with patch.object(Inferencer, "__init__", return_value=None): + pipeline = AutoPipeline.get_pipeline("inferencer", model_args, dataset_args, inferencer_args) self.assertTrue(isinstance(pipeline, Inferencer)) diff --git a/tests/pipeline/test_finetuner_distributed_loss.py b/tests/pipeline/test_finetuner_distributed_loss.py index b84831abe..81dab60a4 100644 --- a/tests/pipeline/test_finetuner_distributed_loss.py +++ b/tests/pipeline/test_finetuner_distributed_loss.py @@ -137,6 +137,7 @@ class PeftMethod(Enum): class TestDtype(Enum): + __test__ = False FP32 = "fp32" BF16 = "bf16" diff --git a/tests/pipeline/test_memory_safe_vllm_inferencer.py b/tests/pipeline/test_memory_safe_vllm_inferencer.py index afa688b4d..cf0d2e4ec 100644 --- a/tests/pipeline/test_memory_safe_vllm_inferencer.py +++ b/tests/pipeline/test_memory_safe_vllm_inferencer.py @@ -4,6 +4,8 @@ import pytest +pytestmark = [pytest.mark.optional_backend, pytest.mark.gpu, pytest.mark.online] + vllm = pytest.importorskip("vllm") from lmflow.args import DatasetArguments, InferencerArguments, ModelArguments diff --git a/tests/pipeline/test_sglang_infernecer.py b/tests/pipeline/test_sglang_infernecer.py index 42f2912e4..6c82c9607 100644 --- a/tests/pipeline/test_sglang_infernecer.py +++ b/tests/pipeline/test_sglang_infernecer.py @@ -1,10 +1,10 @@ import numpy as np import pytest -pytest.importorskip("sglang") +pytestmark = [pytest.mark.optional_backend, pytest.mark.gpu, pytest.mark.online] -from sglang.srt.entrypoints.engine import Engine -from sglang.srt.server_args import ServerArgs +Engine = pytest.importorskip("sglang.srt.entrypoints.engine", exc_type=ImportError).Engine +ServerArgs = pytest.importorskip("sglang.srt.server_args", exc_type=ImportError).ServerArgs from lmflow.args import InferencerArguments, ModelArguments from lmflow.datasets.dataset import Dataset diff --git a/tests/pipeline/test_vllm_inferencer.py b/tests/pipeline/test_vllm_inferencer.py index 7aaa512fd..70cb09763 100644 --- a/tests/pipeline/test_vllm_inferencer.py +++ b/tests/pipeline/test_vllm_inferencer.py @@ -5,6 +5,8 @@ import numpy as np import pytest +pytestmark = pytest.mark.optional_backend + from lmflow.args import DatasetArguments, InferencerArguments, ModelArguments from lmflow.utils.protocol import DataProto diff --git a/tests/utils/test_conversation_template.py b/tests/utils/test_conversation_template.py index bc93b4a37..b0bafe80a 100644 --- a/tests/utils/test_conversation_template.py +++ b/tests/utils/test_conversation_template.py @@ -1,9 +1,12 @@ import unittest +import pytest from transformers import AutoTokenizer from lmflow.utils.conversation_template import PRESET_TEMPLATES +pytestmark = pytest.mark.online + CONVERSATION_SINGLETURN = { "system": "sysinfo", "messages": [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi!"}], diff --git a/tests/utils/test_data_utils.py b/tests/utils/test_data_utils.py index bd615fa8d..a8bcc224e 100644 --- a/tests/utils/test_data_utils.py +++ b/tests/utils/test_data_utils.py @@ -1,4 +1,7 @@ #!/bin/env/python3 +import json +import os +import tempfile import unittest from lmflow.utils.data_utils import answer_extraction, batchlize, load_data @@ -54,10 +57,22 @@ class DataUtilsTest(unittest.TestCase): - def test_load_data(self): - file_name = "data/example_dataset/test/test_13.json" + def load_fixture(self): + with tempfile.TemporaryDirectory() as temp_dir: + file_name = os.path.join(temp_dir, "test.json") + fixture = { + "type": "text2text", + "instances": [ + {"input": input_text, "output": output_text} + for input_text, output_text in zip(groundtruth_inputs, groundtruth_outputs) + ], + } + with open(file_name, "w", encoding="utf-8") as fout: + json.dump(fixture, fout) + return load_data(file_name=file_name) - inputs, outputs, datasize = load_data(file_name=file_name) + def test_load_data(self): + inputs, outputs, datasize = self.load_fixture() # Test for inputs for i in range(0, len(inputs)): self.assertEqual(inputs[i], groundtruth_inputs[i]) @@ -68,8 +83,7 @@ def test_load_data(self): self.assertEqual(datasize, 13) def test_batchlize(self): - file_name = "data/example_dataset/test/test_13.json" - inputs, outputs, datasize = load_data(file_name=file_name) + inputs, outputs, _ = self.load_fixture() dataset = [] for idx in range(len(outputs)): dataset.append({"input": inputs[idx], "output": outputs[idx], "input_idx": idx})