chore: clarify dependency requirement scopes

.github/workflows/benchmark.yml

@@ -27,7 +27,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[nlp]"
-          pip install -r requirements-dev.txt
+          pip install -r requirements-benchmark.txt
           # Verify pytest-benchmark is installed and working
           python -c "import pytest_benchmark; print('pytest-benchmark version:', pytest_benchmark.__version__)"
           python -m pytest --version

.github/workflows/ci.yml

@@ -47,26 +47,25 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
 
-      - name: Install base tooling
+      - name: Upgrade pip
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pytest-cov coverage
 
       - name: Install dependencies (core)
         if: matrix.install-profile == 'core'
         run: |
-          pip install -e ".[dev,cli]"
+          pip install -e ".[test,cli]" -r requirements-test.txt
 
       - name: Install dependencies (nlp)
         if: matrix.install-profile == 'nlp'
         run: |
-          pip install -e ".[dev,cli,nlp]"
+          pip install -e ".[test,cli,nlp]" -r requirements-test.txt
           python -m spacy download en_core_web_sm
 
       - name: Install dependencies (nlp-advanced)
         if: matrix.install-profile == 'nlp-advanced'
         run: |
-          pip install -e ".[dev,cli,nlp,nlp-advanced]"
+          pip install -e ".[test,cli,nlp,nlp-advanced]" -r requirements-test.txt
           python -m spacy download en_core_web_sm
 
       - name: Run tests (core)

.github/workflows/release.yml

@@ -126,8 +126,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[all,dev]"
-          pip install -r requirements-dev.txt
+          pip install -e ".[all,test]"
+          pip install -r requirements-test.txt
           pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
 
       - name: Run tests with segfault protection
@@ -158,8 +158,7 @@ jobs:
       - name: Install core + CLI dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pytest-cov coverage
-          pip install -e ".[dev,cli]"
+          pip install -e ".[test,cli]" -r requirements-test.txt
 
       - name: Run Python 3.13 core + CLI tests
         run: |

CONTRIBUTING.md

@@ -36,6 +36,13 @@ python -m pip install --upgrade pip
 pip install -e ".[dev,cli]"
 ```
 
+For pinned local tooling, install the development requirements after the editable
+package install:
+
+```bash
+pip install -r requirements-dev.txt
+```
+
 For optional NLP or OCR work, install the relevant extras:
 
 ```bash

README.md

@@ -162,5 +162,6 @@ cd datafog-python
 python -m venv .venv
 source .venv/bin/activate  # Windows: .venv\Scripts\activate
 pip install -e ".[all,dev]"
+pip install -r requirements-dev.txt
 pytest tests/
 ```

justfile

@@ -21,8 +21,8 @@ default:
 # Set up the development environment
 setup:
     python -m venv {{venv_dir}}
-    {{pip}} install -e ".[dev]"
-    {{pip}} install isort black blacken-docs flake8 tox coverage pytest pytest-cov
+    {{pip}} install -e ".[dev,cli]"
+    {{pip}} install -r requirements-dev.txt
 
 # Format the code
 format:
@@ -79,4 +79,4 @@ upload: clean
 publish: tag upload
 
 # Run all checks (format, lint, test, coverage)
-check: format lint test coverage
+check: format lint test coverage

requirements-benchmark.txt

@@ -0,0 +1,3 @@
+# Benchmark-only dependencies.
+-r requirements-test.txt
+pytest-benchmark==4.0.0

requirements-dev.txt

@@ -1,7 +1,11 @@
-# Include the base requirements
--r requirements.txt
+# Local development tooling.
+#
+# Install the package itself with the extras you need, for example:
+#   pip install -e ".[dev,cli]"
+#   pip install -e ".[all,dev]"
+-r requirements-test.txt
+-r requirements-docs.txt
 
-# Development and testing dependencies
 just==0.8.162
 isort==5.13.2
 black==26.3.1
@@ -14,4 +18,4 @@ pytest-benchmark==4.0.0
 mypy==1.9.0
 autoflake==2.2.1
 pre-commit==3.7.0
-ruff==0.3.4
+ruff==0.3.4

requirements-docs.txt

@@ -0,0 +1,2 @@
+# Documentation-only dependencies.
+sphinx==7.2.6

requirements-test.txt

@@ -0,0 +1,4 @@
+# Test-only dependencies.
+pytest==9.0.3
+pytest-asyncio==1.3.0
+pytest-cov==7.1.0

requirements.txt

@@ -1,23 +1,7 @@
-# Core dependencies with pinned versions
-pandas==2.2.3
-requests==2.33.0
-spacy==3.7.5
+# Minimal runtime dependencies for local installs.
+#
+# Published package metadata in setup.py is the source of truth. Optional
+# feature dependencies live behind package extras such as nlp, ocr, web, and all.
 pydantic==2.11.4
-Pillow==12.2.0
-sentencepiece==0.2.1
-protobuf==6.33.5
-pytesseract==0.3.13
-aiohttp==3.13.4
-numpy==1.26.4
-fastapi==0.115.12
-asyncio==3.4.3
-setuptools>=68.0.0
 pydantic-settings==2.3.4
-typer==0.12.3
-sphinx==7.2.6
-cryptography==46.0.7
-
-# Testing dependencies
-pytest==9.0.3
-pytest-asyncio==0.21.0
-pytest-cov==4.1.0
+typing-extensions>=4.0

setup.py

@@ -24,64 +24,80 @@
 ]
 
 # Optional heavy dependencies
+nlp_deps = [
+    "spacy>=3.7.0,<4.0",
+]
+
+nlp_advanced_deps = [
+    "gliner>=0.2.5",
+    "torch>=2.1.0,<2.7",
+    "transformers>=4.20.0",
+    "huggingface-hub>=0.16.0",
+]
+
+ocr_deps = [
+    "pytesseract>=0.3.0",
+    "Pillow>=12.2.0",
+    "sentencepiece>=0.2.0",
+    "protobuf>=4.0.0",
+]
+
+distributed_deps = [
+    "pandas>=2.0.0",
+    "numpy>=1.24.0",
+]
+
+web_deps = [
+    "fastapi>=0.100.0",
+    "aiohttp>=3.13.4",
+    "requests>=2.33.0",
+]
+
+cli_deps = [
+    "typer>=0.12.0",
+    "pydantic-settings>=2.0.0",
+]
+
+crypto_deps = [
+    "cryptography>=46.0.7",
+]
+
+test_deps = [
+    "pytest>=9.0.3",
+    "pytest-asyncio>=1.3.0",
+    "pytest-cov>=7.1.0",
+]
+
+docs_deps = [
+    "sphinx>=7.2.6",
+]
+
+benchmark_deps = [
+    "pytest-benchmark>=4.0.0",
+]
+
 extras_require = {
-    "nlp": [
-        "spacy>=3.7.0,<4.0",
-    ],
-    "nlp-advanced": [
-        "gliner>=0.2.5",
-        "torch>=2.1.0,<2.7",
-        "transformers>=4.20.0",
-        "huggingface-hub>=0.16.0",
-    ],
-    "ocr": [
-        "pytesseract>=0.3.0",
-        "Pillow>=10.0.0",
-        "sentencepiece>=0.2.0",
-        "protobuf>=4.0.0",
-    ],
-    "distributed": [
-        "pandas>=2.0.0",
-        "numpy>=1.24.0",
-    ],
-    "web": [
-        "fastapi>=0.100.0",
-        "aiohttp>=3.8.0",
-        "requests>=2.30.0",
-    ],
-    "cli": [
-        "typer>=0.12.0",
-        "pydantic-settings>=2.0.0",
-    ],
-    "crypto": [
-        "cryptography>=40.0.0",
-    ],
-    "dev": [
-        "pytest>=7.0.0",
-        "pytest-asyncio>=0.21.0",
-        "pytest-cov>=4.0.0",
-        "sphinx>=7.0.0",
-    ],
+    "nlp": nlp_deps,
+    "nlp-advanced": nlp_advanced_deps,
+    "ocr": ocr_deps,
+    "distributed": distributed_deps,
+    "web": web_deps,
+    "cli": cli_deps,
+    "crypto": crypto_deps,
+    "test": test_deps,
+    "docs": docs_deps,
+    "benchmark": benchmark_deps,
+    "dev": test_deps + docs_deps,
     # Convenience bundles
-    "all": [
-        "spacy>=3.7.0,<4.0",
-        "gliner>=0.2.5",
-        "torch>=2.1.0,<2.7",
-        "transformers>=4.20.0",
-        "huggingface-hub>=0.16.0",
-        "pytesseract>=0.3.0",
-        "Pillow>=10.0.0",
-        "sentencepiece>=0.2.0",
-        "protobuf>=4.0.0",
-        "pandas>=2.0.0",
-        "numpy>=1.24.0",
-        "fastapi>=0.100.0",
-        "aiohttp>=3.8.0",
-        "requests>=2.30.0",
-        "typer>=0.12.0",
-        "pydantic-settings>=2.0.0",
-        "cryptography>=40.0.0",
-    ],
+    "all": (
+        nlp_deps
+        + nlp_advanced_deps
+        + ocr_deps
+        + distributed_deps
+        + web_deps
+        + cli_deps
+        + crypto_deps
+    ),
 }
 
 setup(

tox.ini

@@ -4,9 +4,11 @@ isolated_build = True
 
 [testenv:integration]
 deps =
-    -r requirements-dev.txt
+    -r requirements-test.txt
     pyspark>=3.0.0
-extras = all
+extras =
+    all
+    test
 allowlist_externals =
     tesseract
     pip
@@ -16,8 +18,10 @@ commands =
 
 [testenv]
 deps =
-    -r requirements-dev.txt
-extras = all
+    -r requirements-test.txt
+extras =
+    all
+    test
 allowlist_externals =
     tesseract
     pip
@@ -29,17 +33,17 @@ commands =
 [testenv:lint]
 skip_install = true
 deps =
-    black
-    isort
-    flake8
+    black==26.3.1
+    isort==5.13.2
+    flake8==7.0.0
 commands =
     black --check .
     isort --check-only .
     flake8 .
 
 [testenv:typecheck]
 deps =
-    mypy
+    mypy==1.9.0
 commands =
     mypy datafog tests