diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml new file mode 100644 index 00000000..4134fb3f --- /dev/null +++ b/.github/workflows/build_package.yml @@ -0,0 +1,50 @@ +name: Build Package + +on: + pull_request: + push: + branches: + - release/* + +jobs: + publish: + runs-on: ubuntu-latest + strategy: + matrix: + package: + - ragna + - ragna-base + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build toml + + - name: Modify pyproject.toml + run: python scripts/build_helper.py ${{ matrix.package }} + + - name: Build distribution + run: | + echo "Building package: ${{ matrix.package }}" + python -m build + echo "Build complete for ${{ matrix.package }}" + + - name: Verify package + run: | + echo "Verifying package: ${{ matrix.package }}" + pip install dist/*.whl + pip check + + - name: Visualize dependency tree of built package + run: | + pip install pipdeptree + pipdeptree -d 1 diff --git a/pyproject.toml b/pyproject.toml index 9ceb1569..9284cd3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ - "setuptools>=45", - "setuptools_scm[toml]>=6.2", + "setuptools>=66", + "setuptools_scm[toml]>=8", ] build-backend = "setuptools.build_meta" @@ -20,29 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] requires-python = ">=3.9" -dependencies = [ - "aiofiles", - "emoji", - "fastapi", - "httpx", - "importlib_metadata>=4.6; python_version<'3.10'", - "packaging", - "panel==1.4.2", - "pydantic>=2", - "pydantic-core", - "pydantic-settings>=2", - "PyJWT", - "python-multipart", - "redis", - "questionary", - "rich", - "sqlalchemy>=2", - "starlette", - "tomlkit", - "typer", - "uvicorn", -] -dynamic = ["version"] +dynamic = ["dependencies", "version"] [project.urls] Homepage = "https://ragna.chat" @@ -50,19 +28,8 @@ Documentation = "https://ragna.chat" Changelog = "https://ragna.chat/en/stable/references/release-notes/" Repository = "https://github.com/Quansight/ragna" -[project.optional-dependencies] -# to update the array below, run scripts/update_optional_dependencies.py -all = [ - "chromadb>=0.4.13", - "httpx_sse", - "ijson", - "lancedb>=0.2", - "pyarrow", - "pymupdf>=1.23.6", - "python-docx", - "python-pptx", - "tiktoken", -] +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} [tool.setuptools_scm] write_to = "ragna/_version.py" diff --git a/requirements-base.txt b/requirements-base.txt new file mode 100644 index 00000000..bdc51b3f --- /dev/null +++ b/requirements-base.txt @@ -0,0 +1,20 @@ +aiofiles +emoji +fastapi +httpx +importlib_metadata>=4.6; python_version<'3.10' +packaging +panel==1.4.2 +pydantic>=2 +pydantic-core +pydantic-settings>=2 +PyJWT +python-multipart +redis +questionary +rich +sqlalchemy>=2 +starlette +tomlkit +typer +uvicorn diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..b225b278 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +ragna-base +chromadb>=0.4.13 +httpx_sse +ijson +lancedb>=0.2 +pyarrow +pymupdf>=1.23.6 +python-docx +python-pptx +tiktoken diff --git a/scripts/build_helper.py b/scripts/build_helper.py new file mode 100644 index 00000000..cad214d9 --- /dev/null +++ b/scripts/build_helper.py @@ -0,0 +1,34 @@ +import sys +from pathlib import Path + +import toml + +HERE = Path(__file__).parent +PROJECT_ROOT = HERE.parent +PYPROJECT_TOML = PROJECT_ROOT / "pyproject.toml" + + +def modify_pyproject(package_name): + with open(PYPROJECT_TOML, "r") as f: + pyproject_data = toml.load(f) + + pyproject_data["project"]["name"] = package_name + + pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = { + "file": [ + "requirements-base.txt" + if package_name == "ragna-base" + else "requirements.txt" + ] + } + + with open(PYPROJECT_TOML, "w") as f: + toml.dump(pyproject_data, f) + + +if __name__ == "__main__": + package_name = sys.argv[1] + if package_name not in ["ragna", "ragna-base"]: + print("Invalid package name. Must be 'ragna' or 'ragna-base'.") + sys.exit(1) + modify_pyproject(package_name) diff --git a/scripts/update_optional_dependencies.py b/scripts/update_optional_dependencies.py index a99f206c..768e722e 100644 --- a/scripts/update_optional_dependencies.py +++ b/scripts/update_optional_dependencies.py @@ -2,15 +2,15 @@ from functools import reduce from pathlib import Path -import tomlkit -import tomlkit.items from packaging.requirements import Requirement import ragna from ragna.core import Assistant, SourceStorage HERE = Path(__file__).parent -PYPROJECT_TOML = HERE / ".." / "pyproject.toml" +PROJECT_ROOT = HERE.parent +REQUIREMENTS_TXT = PROJECT_ROOT / "requirements.txt" +REQUIREMENTS_BASE_TXT = PROJECT_ROOT / "requirements-base.txt" def main(): @@ -18,7 +18,7 @@ def main(): extract_builtin_document_handler_requirements(), extract_builtin_component_requirements(), ) - update_pyproject_toml(optional_dependencies) + create_requirements_txt(optional_dependencies) def make_optional_dependencies(*optional_requirements): @@ -57,18 +57,23 @@ def extract_builtin_component_requirements(): return dict(requirements) -def update_pyproject_toml(optional_dependencies): - with open(PYPROJECT_TOML) as file: - document = tomlkit.load(file) +def create_requirements_txt(optional_dependencies): + # Read existing dependencies from requirements-base.txt + existing_dependencies = set() + if REQUIREMENTS_BASE_TXT.exists(): + with open(REQUIREMENTS_BASE_TXT, "r") as file: + existing_dependencies = set(line.strip() for line in file) - document["project"]["optional-dependencies"]["all"] = tomlkit.items.Array( - list(map(tomlkit.items.String.from_raw, optional_dependencies)), - trivia=tomlkit.items.Trivia(), - multiline=True, - ) + # Add optional dependencies + all_dependencies = existing_dependencies.union(optional_dependencies) + + # Sort the dependencies to maintain a consistent order + sorted_dependencies = sorted(all_dependencies) - with open(PYPROJECT_TOML, "w") as file: - tomlkit.dump(document, file) + # Overwrite the requirements.txt file with the sorted dependencies + with open(REQUIREMENTS_TXT, "w") as file: + file.write("\n".join(sorted_dependencies)) + file.write("\n") def append_version_specifiers(version_specifiers, obj):