[ci] establish baseline CI/CD checks

.github/workflows/ci.yaml

@@ -0,0 +1,98 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  PYTHONDONTWRITEBYTECODE: "1"
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.2.0
+        with:
+          python-version: "3.11"
+      - name: Install Ruff
+        run: python -m pip install ruff==0.11.4
+      - name: Run high-confidence lint checks
+        run: ruff check src tests setup.py --select E4,E7,E9,F --ignore E731
+
+  unit-tests:
+    name: CPU unit tests (Python 3.9)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      HF_HUB_OFFLINE: "1"
+      TRANSFORMERS_OFFLINE: "1"
+      WANDB_MODE: disabled
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.2.0
+        with:
+          python-version: "3.9"
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            setup.py
+      - name: Install LMFlow and test dependencies
+        run: python -m pip install -e ".[develop]"
+      - name: Run offline CPU tests
+        run: >-
+          python -m pytest -q --strict-markers
+          -m "not gpu and not slow and not online and not optional_backend"
+
+  package:
+    name: Build package
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.2.0
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install build tools
+        run: python -m pip install build twine
+      - name: Build distributions
+        run: python -m build
+      - name: Validate distributions
+        run: python -m twine check dist/*
+      - name: Install wheel without runtime dependencies
+        run: python -m pip install --no-deps --force-reinstall dist/*.whl
+      - name: Verify installed package metadata
+        run: |
+          python - <<'PY'
+          from importlib.metadata import metadata, version
+
+          package_metadata = metadata("lmflow")
+          assert version("lmflow")
+          assert package_metadata["Requires-Python"] == ">=3.9"
+          PY
+      - name: Upload distributions
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: python-distributions
+          path: dist/
+          if-no-files-found: error

.github/workflows/documentation.yaml

@@ -1,24 +1,59 @@
 name: Docs
-on: [push, pull_request, workflow_dispatch]
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: pages-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  docs:
+  build:
+    name: Build documentation
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
-      - name: python environment setup
-        uses: actions/setup-python@v5.1.0
-        with: 
+      - name: Check out repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.2.0
+        with:
           python-version: "3.11"
+          cache: pip
+          cache-dependency-path: docs/requirements.txt
       - name: Install dependencies
-        run: |
-          pip install -r ./docs/requirements.txt
-      - name: Sphinx build
-        run: |
-          sphinx-build docs/source _build
-      - name: Deploy
-        uses: peaceiris/actions-gh-pages@v3
+        run: python -m pip install -r docs/requirements.txt
+      - name: Build documentation
+        run: sphinx-build -b html docs/source _build/html
+      - name: Configure GitHub Pages
+        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0
+      - name: Upload Pages artifact
+        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+        uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0
         with:
-          publish_branch: gh-pages
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: _build/
-          force_orphan: true
+          path: _build/html
+
+  deploy:
+    name: Deploy documentation
+    if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5

pyproject.toml

@@ -51,6 +51,19 @@ ignore = [
 
 [tool.ruff.lint.per-file-ignores]
 "tests/**/*.py" = ["F841"]
+"tests/pipeline/test_sglang_infernecer.py" = ["E402"]
+"tests/pipeline/test_vllm_inferencer.py" = ["E402"]
 "src/lmflow/utils/conversation_template/*.py" = ["E501"]
 "src/lmflow/utils/llava_conversation_lib.py" = ["E501"]
 "src/lmflow/utils/constants.py" = ["E501"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "gpu: requires a GPU",
+    "slow: takes substantially longer than the unit-test suite",
+    "online: requires network access or a remote model/dataset",
+    "optional_backend: requires an optional inference backend",
+    "lmflow_core: tests for core LMFlow functionality",
+    "dothis: mark for developers to run explicitly",
+]

scripts/run_unittest.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-python -m unittest discover
+python -m pytest -q --strict-markers \
+  -m "not gpu and not slow and not online and not optional_backend"

setup.py

@@ -56,7 +56,7 @@
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
     ],
-    requires_python=">=3.9",
+    python_requires=">=3.9",
 )
 
 # optionals

src/lmflow/pipeline/inferencer.py

@@ -5,6 +5,7 @@
 import logging
 import os
 import subprocess
+import sys
 
 import numpy as np
 import torch
@@ -626,7 +627,7 @@ def inference(
 
     def code_exec(self, code):
         # Execute the code
-        result = subprocess.run(["python", "-c", code], capture_output=True, text=True)
+        result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
 
         # Print the result
         if result.returncode == 0:

src/lmflow/pipeline/sglang_inferencer.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python
 # Copyright 2024 Statistics and Machine Learning Research Group. All rights reserved.
-import json
 import logging
 import os
-from typing import Optional, Union
+from typing import Optional
 
 from transformers import AutoTokenizer
 

tests/datasets/test_dataset.py

@@ -14,24 +14,33 @@
 
 import json
 import os
+import tempfile
 import unittest
-from pathlib import Path
 
 from lmflow.args import DatasetArguments
 from lmflow.datasets.dataset import Dataset
 
 
 class DatasetTest(unittest.TestCase):
     def test_init(self):
-        dataset_dir = "data/example_dataset/train"
-        data_args = DatasetArguments(dataset_path=dataset_dir)
-        dataset = Dataset(data_args, backend="huggingface")
-        hf_dataset = dataset.get_backend_dataset()
-
-        with open(os.path.join(Path(dataset_dir), "train_50.json")) as fin:
-            json_obj = json.load(fin)
-            for i in range(len(hf_dataset)):
-                self.assertEqual(json_obj["instances"][i], hf_dataset[i])
+        json_obj = {
+            "type": "text2text",
+            "instances": [
+                {"input": "INPUT 1", "output": "OUTPUT 1"},
+                {"input": "INPUT 2", "output": "OUTPUT 2"},
+            ],
+        }
+        with tempfile.TemporaryDirectory() as dataset_dir:
+            with open(os.path.join(dataset_dir, "train.json"), "w", encoding="utf-8") as fout:
+                json.dump(json_obj, fout)
+
+            data_args = DatasetArguments(dataset_path=dataset_dir)
+            dataset = Dataset(data_args, backend="huggingface")
+            hf_dataset = dataset.get_backend_dataset()
+
+            self.assertEqual(len(hf_dataset), len(json_obj["instances"]))
+            for expected, actual in zip(json_obj["instances"], hf_dataset):
+                self.assertEqual(expected, actual)
 
     def test_create_from_dict(self):
         data_dict = {

tests/models/test_hf_decoder_model.py

@@ -19,6 +19,8 @@
 import os
 from pathlib import Path
 
+import pytest
+
 from lmflow.args import DatasetArguments, ModelArguments
 from lmflow.datasets.dataset import Dataset
 from lmflow.models.hf_decoder_model import HFDecoderModel
@@ -28,6 +30,8 @@
 )
 from lmflow.utils.conversation_template import PRESET_TEMPLATES
 
+pytestmark = pytest.mark.online
+
 
 SAMPLE_TEXT = "Defintion: In this task, we ask you to write an answer to a question that involves events that may be stationary (not changing over time) or transient (changing over time). For example, the sentence \"he was born in the U.S.\" contains a stationary event since it will last forever; however, \"he is hungry\" contains a transient event since it will remain true for a short period of time. Note that a lot of the questions could have more than one correct answer. We only need a single most-likely answer. Please try to keep your \"answer\" as simple as possible. Concise and simple \"answer\" is preferred over those complex and verbose ones. \\n Input: Sentence: It's hail crackled across the comm, and Tara spun to retake her seat at the helm. \nQuestion: Will the hail storm ever end? \\n Output: NA \\n\\n"
 
@@ -671,4 +675,4 @@ def test_inference(self):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

tests/models/test_tool_inferencer.py

@@ -1,6 +1,5 @@
 import unittest
 
-from lmflow.args import DatasetArguments, InferencerArguments, ModelArguments
 from lmflow.pipeline.inferencer import ToolInferencer
 
 CODE_1 = 'print("hello world")'
@@ -13,11 +12,10 @@
 
 
 class ToolInferencerTest(unittest.TestCase):
-    def set_up(self):
-        model_args = ModelArguments(model_name_or_path="codellama/CodeLlama-7b-instruct-hf")
-        inferencer_args = InferencerArguments()
-        data_args = DatasetArguments()
-        self.toolinf = ToolInferencer(model_args, data_args, inferencer_args)
+    def setUp(self):
+        # code_exec does not use model state; bypass model initialization so this
+        # remains a fast, offline unit test.
+        self.toolinf = object.__new__(ToolInferencer)
 
     def test_code_exec_1(self, code=CODE_1, expected_output=RES_1):
         toolinf_res = self.toolinf.code_exec(code)

tests/pipeline/test_auto_pipeline.py

@@ -1,4 +1,5 @@
 import unittest
+from unittest.mock import patch
 
 from lmflow.args import DatasetArguments, EvaluatorArguments, FinetunerArguments, InferencerArguments, ModelArguments
 from lmflow.pipeline.auto_pipeline import AutoPipeline
@@ -14,23 +15,26 @@ def test_get_evaluator_pipeline(self):
         model_args = ModelArguments(model_name_or_path=MODEL_NAME)
         dataset_args = DatasetArguments()
         evaluator_args = EvaluatorArguments()
-        pipeline = AutoPipeline.get_pipeline("evaluator", model_args, dataset_args, evaluator_args)
+        with patch.object(Evaluator, "__init__", return_value=None):
+            pipeline = AutoPipeline.get_pipeline("evaluator", model_args, dataset_args, evaluator_args)
 
         self.assertTrue(isinstance(pipeline, Evaluator))
 
     def test_get_finetuner_pipeline(self):
         model_args = ModelArguments(model_name_or_path=MODEL_NAME)
         dataset_args = DatasetArguments()
         finetuner_args = FinetunerArguments(output_dir="~/tmp")
-        pipeline = AutoPipeline.get_pipeline("finetuner", model_args, dataset_args, finetuner_args)
+        with patch.object(Finetuner, "__init__", return_value=None):
+            pipeline = AutoPipeline.get_pipeline("finetuner", model_args, dataset_args, finetuner_args)
 
         self.assertTrue(isinstance(pipeline, Finetuner))
 
     def test_get_inferencer_pipeline(self):
         model_args = ModelArguments(model_name_or_path=MODEL_NAME)
         dataset_args = DatasetArguments()
         inferencer_args = InferencerArguments()
-        pipeline = AutoPipeline.get_pipeline("inferencer", model_args, dataset_args, inferencer_args)
+        with patch.object(Inferencer, "__init__", return_value=None):
+            pipeline = AutoPipeline.get_pipeline("inferencer", model_args, dataset_args, inferencer_args)
 
         self.assertTrue(isinstance(pipeline, Inferencer))
 

tests/pipeline/test_finetuner_distributed_loss.py

@@ -137,6 +137,7 @@ class PeftMethod(Enum):
 
 
 class TestDtype(Enum):
+    __test__ = False
     FP32 = "fp32"
     BF16 = "bf16"
 

tests/pipeline/test_memory_safe_vllm_inferencer.py

@@ -4,6 +4,8 @@
 
 import pytest
 
+pytestmark = [pytest.mark.optional_backend, pytest.mark.gpu, pytest.mark.online]
+
 vllm = pytest.importorskip("vllm")
 
 from lmflow.args import DatasetArguments, InferencerArguments, ModelArguments

tests/pipeline/test_sglang_infernecer.py

@@ -1,10 +1,10 @@
 import numpy as np
 import pytest
 
-pytest.importorskip("sglang")
+pytestmark = [pytest.mark.optional_backend, pytest.mark.gpu, pytest.mark.online]
 
-from sglang.srt.entrypoints.engine import Engine
-from sglang.srt.server_args import ServerArgs
+Engine = pytest.importorskip("sglang.srt.entrypoints.engine", exc_type=ImportError).Engine
+ServerArgs = pytest.importorskip("sglang.srt.server_args", exc_type=ImportError).ServerArgs
 
 from lmflow.args import InferencerArguments, ModelArguments
 from lmflow.datasets.dataset import Dataset