PoC for ragna-base package with just base dependencies

.github/workflows/build_package.yml

@@ -0,0 +1,50 @@
+name: Build Package
+
+on:
+  pull_request:
+  push:
+    branches:
+      - release/*
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        package:
+          - ragna
+          - ragna-base
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build toml
+
+      - name: Modify pyproject.toml
+        run: python scripts/build_helper.py ${{ matrix.package }}
+
+      - name: Build distribution
+        run: |
+          echo "Building package: ${{ matrix.package }}"
+          python -m build
+          echo "Build complete for ${{ matrix.package }}"
+
+      - name: Verify package
+        run: |
+          echo "Verifying package: ${{ matrix.package }}"
+          pip install dist/*.whl
+          pip check
+
+      - name: Visualize dependency tree of built package
+        run: |
+          pip install pipdeptree
+          pipdeptree -d 1

pyproject.toml

@@ -1,7 +1,7 @@
 [build-system]
 requires = [
-    "setuptools>=45",
-    "setuptools_scm[toml]>=6.2",
+    "setuptools>=66",
+    "setuptools_scm[toml]>=8",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -20,49 +20,16 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 requires-python = ">=3.9"
-dependencies = [
-    "aiofiles",
-    "emoji",
-    "fastapi",
-    "httpx",
-    "importlib_metadata>=4.6; python_version<'3.10'",
-    "packaging",
-    "panel==1.4.2",
-    "pydantic>=2",
-    "pydantic-core",
-    "pydantic-settings>=2",
-    "PyJWT",
-    "python-multipart",
-    "redis",
-    "questionary",
-    "rich",
-    "sqlalchemy>=2",
-    "starlette",
-    "tomlkit",
-    "typer",
-    "uvicorn",
-]
-dynamic = ["version"]
+dynamic = ["dependencies", "version"]
 
 [project.urls]
 Homepage = "https://ragna.chat"
 Documentation = "https://ragna.chat"
 Changelog = "https://ragna.chat/en/stable/references/release-notes/"
 Repository = "https://github.com/Quansight/ragna"
 
-[project.optional-dependencies]
-# to update the array below, run scripts/update_optional_dependencies.py
-all = [
-    "chromadb>=0.4.13",
-    "httpx_sse",
-    "ijson",
-    "lancedb>=0.2",
-    "pyarrow",
-    "pymupdf>=1.23.6",
-    "python-docx",
-    "python-pptx",
-    "tiktoken",
-]
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
 
 [tool.setuptools_scm]
 write_to = "ragna/_version.py"

requirements-base.txt

@@ -0,0 +1,20 @@
+aiofiles
+emoji
+fastapi
+httpx
+importlib_metadata>=4.6; python_version<'3.10'
+packaging
+panel==1.4.2
+pydantic>=2
+pydantic-core
+pydantic-settings>=2
+PyJWT
+python-multipart
+redis
+questionary
+rich
+sqlalchemy>=2
+starlette
+tomlkit
+typer
+uvicorn

requirements.txt

@@ -0,0 +1,29 @@
+PyJWT
+aiofiles
+chromadb>=0.4.13
+emoji
+fastapi
+httpx
+httpx_sse
+ijson
+importlib_metadata>=4.6; python_version<'3.10'
+lancedb>=0.2
+packaging
+panel==1.4.2
+pyarrow
+pydantic-core
+pydantic-settings>=2
+pydantic>=2
+pymupdf>=1.23.6
+python-docx
+python-multipart
+python-pptx
+questionary
+redis
+rich
+sqlalchemy>=2
+starlette
+tiktoken
+tomlkit
+typer
+uvicorn

scripts/build_helper.py

@@ -0,0 +1,34 @@
+import sys
+from pathlib import Path
+
+import toml
+
+HERE = Path(__file__).parent
+PROJECT_ROOT = HERE.parent
+PYPROJECT_TOML = PROJECT_ROOT / "pyproject.toml"
+
+
+def modify_pyproject(package_name):
+    with open(PYPROJECT_TOML, "r") as f:
+        pyproject_data = toml.load(f)
+
+    pyproject_data["project"]["name"] = package_name
+
+    pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = {
+        "file": [
+            "requirements-base.txt"
+            if package_name == "ragna-base"
+            else "requirements.txt"
+        ]
+    }
+
+    with open(PYPROJECT_TOML, "w") as f:
+        toml.dump(pyproject_data, f)
+
+
+if __name__ == "__main__":
+    package_name = sys.argv[1]
+    if package_name not in ["ragna", "ragna-base"]:
+        print("Invalid package name. Must be 'ragna' or 'ragna-base'.")
+        sys.exit(1)
+    modify_pyproject(package_name)

scripts/update_optional_dependencies.py

@@ -2,23 +2,23 @@
 from functools import reduce
 from pathlib import Path
 
-import tomlkit
-import tomlkit.items
 from packaging.requirements import Requirement
 
 import ragna
 from ragna.core import Assistant, SourceStorage
 
 HERE = Path(__file__).parent
-PYPROJECT_TOML = HERE / ".." / "pyproject.toml"
+PROJECT_ROOT = HERE.parent
+REQUIREMENTS_TXT = PROJECT_ROOT / "requirements.txt"
+REQUIREMENTS_BASE_TXT = PROJECT_ROOT / "requirements-base.txt"
 
 
 def main():
     optional_dependencies = make_optional_dependencies(
         extract_builtin_document_handler_requirements(),
         extract_builtin_component_requirements(),
     )
-    update_pyproject_toml(optional_dependencies)
+    create_requirements_txt(optional_dependencies)
 
 
 def make_optional_dependencies(*optional_requirements):
@@ -57,18 +57,23 @@ def extract_builtin_component_requirements():
     return dict(requirements)
 
 
-def update_pyproject_toml(optional_dependencies):
-    with open(PYPROJECT_TOML) as file:
-        document = tomlkit.load(file)
+def create_requirements_txt(optional_dependencies):
+    # Read existing dependencies from requirements-base.txt
+    existing_dependencies = set()
+    if REQUIREMENTS_BASE_TXT.exists():
+        with open(REQUIREMENTS_BASE_TXT, "r") as file:
+            existing_dependencies = set(line.strip() for line in file)
 
-    document["project"]["optional-dependencies"]["all"] = tomlkit.items.Array(
-        list(map(tomlkit.items.String.from_raw, optional_dependencies)),
-        trivia=tomlkit.items.Trivia(),
-        multiline=True,
-    )
+    # Add optional dependencies
+    all_dependencies = existing_dependencies.union(optional_dependencies)
+
+    # Sort the dependencies to maintain a consistent order
+    sorted_dependencies = sorted(all_dependencies)
 
-    with open(PYPROJECT_TOML, "w") as file:
-        tomlkit.dump(document, file)
+    # Overwrite the requirements.txt file with the sorted dependencies
+    with open(REQUIREMENTS_TXT, "w") as file:
+        file.write("\n".join(sorted_dependencies))
+        file.write("\n")
 
 
 def append_version_specifiers(version_specifiers, obj):